hcq/oceanbase
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity
Files
dd142a4f9104b39437ae0aee351fd83ea4cd587f
oceanbase/src/share/partition_table/ob_partition_location_cache.cpp
zn0 dd142a4f91 Fix timeout setting in location cache
2021-08-31 22:07:18 +08:00

4711 lines
180 KiB
C++
Raw Blame History

/**
* Copyright (c) 2021 OceanBase
* OceanBase CE is licensed under Mulan PubL v2.
* You can use this software according to the terms and conditions of the Mulan PubL v2.
* You may obtain a copy of Mulan PubL v2 at:
* http://license.coscl.org.cn/MulanPubL-2.0
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
* See the Mulan PubL v2 for more details.
*/
#define USING_LOG_PREFIX SHARE_PT
#include "ob_partition_location_cache.h"
#include "lib/allocator/ob_mod_define.h"
#include "lib/time/ob_tsc_timestamp.h"
#include "share/ob_worker.h"
#include "lib/stat/ob_diagnose_info.h"
#include "lib/time/ob_time_utility.h"
#include "lib/trace/ob_trace_event.h"
#include "lib/ob_running_mode.h"
#include "lib/mysqlclient/ob_mysql_proxy.h"
#include "common/ob_timeout_ctx.h"
#include "share/config/ob_server_config.h"
#include "share/inner_table/ob_inner_table_schema.h"
#include "share/ob_common_rpc_proxy.h"
#include "share/ob_rs_mgr.h"
#include "share/ob_share_util.h"
#include "share/partition_table/ob_location_update_task.h"
#include "share/partition_table/ob_partition_table_operator.h"
#include "share/partition_table/ob_remote_partition_table_operator.h"
#include "share/ob_alive_server_tracer.h"
#include "ob_replica_filter.h"
#include "share/schema/ob_schema_struct.h"
#include "share/schema/ob_part_mgr_util.h"
#include "share/schema/ob_multi_version_schema_service.h"
#include "share/ob_multi_cluster_util.h"
#include "share/ob_remote_server_provider.h"
#include "share/ob_thread_mgr.h"
#include "share/ob_cluster_version.h"
#include "share/ob_server_blacklist.h"
#include "observer/ob_server_struct.h"
#include "rootserver/ob_rs_async_rpc_proxy.h"
namespace oceanbase {
using namespace common;
using namespace common::hash;
using namespace obrpc;
using namespace share;
using namespace oceanbase::share::schema;
namespace share {
ObLocationCacheKey::ObLocationCacheKey()
: table_id_(common::OB_INVALID_ID), partition_id_(common::OB_INVALID_ID), cluster_id_(common::OB_INVALID_ID)
{}
ObLocationCacheKey::ObLocationCacheKey(const uint64_t table_id, const int64_t partition_id)
: table_id_(table_id), partition_id_(partition_id), cluster_id_(common::OB_INVALID_ID)
{}
ObLocationCacheKey::ObLocationCacheKey(const uint64_t table_id, const int64_t partition_id, const int64_t cluster_id)
: table_id_(table_id), partition_id_(partition_id), cluster_id_(cluster_id)
{}
bool ObLocationCacheKey::operator==(const ObIKVCacheKey& other) const
{
const ObLocationCacheKey& other_key = reinterpret_cast<const ObLocationCacheKey&>(other);
return table_id_ == other_key.table_id_ && partition_id_ == other_key.partition_id_ &&
cluster_id_ == other_key.cluster_id_;
}
bool ObLocationCacheKey::operator!=(const ObIKVCacheKey& other) const
{
const ObLocationCacheKey& other_key = reinterpret_cast<const ObLocationCacheKey&>(other);
return table_id_ != other_key.table_id_ || partition_id_ != other_key.partition_id_ ||
cluster_id_ != other_key.cluster_id_;
}
uint64_t ObLocationCacheKey::get_tenant_id() const
{
// NOTE: location cache isolation is disabled to minimize obproxy memory usage.
return common::OB_SYS_TENANT_ID;
}
uint64_t ObLocationCacheKey::hash() const
{
return common::murmurhash(this, sizeof(*this), 0);
}
uint64_t ObLocationCacheKey::hash_v2() const
{
uint64_t hash_val = 0;
hash_val = murmurhash(&table_id_, sizeof(table_id_), hash_val);
hash_val = murmurhash(&partition_id_, sizeof(partition_id_), hash_val);
hash_val = murmurhash(&cluster_id_, sizeof(cluster_id_), hash_val);
return hash_val;
}
int64_t ObLocationCacheKey::size() const
{
return sizeof(*this);
}
int ObLocationCacheKey::deep_copy(char* buf, const int64_t buf_len, ObIKVCacheKey*& key) const
{
int ret = OB_SUCCESS;
if (NULL == buf || buf_len < size()) {
ret = common::OB_INVALID_ARGUMENT;
LOG_WARN("invalid argument", K(ret), KP(buf), K(buf_len), "size", size());
} else {
ObLocationCacheKey* pkey = new (buf) ObLocationCacheKey();
*pkey = *this;
key = pkey;
}
return ret;
}
ObLocationCacheValue::ObLocationCacheValue() : size_(0), buffer_(NULL)
{}
int64_t ObLocationCacheValue::size() const
{
return static_cast<int64_t>(sizeof(*this) + size_);
}
int ObLocationCacheValue::deep_copy(char* buf, const int64_t buf_len, ObIKVCacheValue*& value) const
{
int ret = OB_SUCCESS;
if (NULL == buf || buf_len < size()) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("invalid argument", K(ret), KP(buf), K(buf_len), "size", size());
} else {
ObLocationCacheValue* pvalue = new (buf) ObLocationCacheValue();
pvalue->size_ = size_;
pvalue->buffer_ = buf + sizeof(*this);
MEMCPY(pvalue->buffer_, buffer_, size_);
value = pvalue;
}
return ret;
}
///////////////////////////////////////////
int ObTenantSqlRenewStat::assign(const ObTenantSqlRenewStat& other)
{
int ret = OB_SUCCESS;
if (!other.is_valid()) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("invalid argument", KR(ret));
} else {
tenant_id_ = other.tenant_id_;
start_sql_renew_ts_ = other.start_sql_renew_ts_;
sql_renew_count_ = other.sql_renew_count_;
}
return ret;
}
///////////////////////////////////////////
int ObTenantStatGuard::set_tenant_stat(const uint64_t tenant_id, TenantSqlRenewInfoMap& map,
common::ObCachedAllocator<ObTenantSqlRenewStat>& object_pool, ObSpinLock& lock)
{
int ret = OB_SUCCESS;
if (OB_INVALID_TENANT_ID == tenant_id) {
ret = OB_INVALID_ARGUMENT;
STORAGE_LOG(WARN, "invalid argument", K(ret), K(tenant_id));
} else if (OB_SUCC(map.get_refactored(tenant_id, tenant_stat_))) {
tenant_stat_->lock_.lock();
tenant_id_ = tenant_id;
} else if (OB_HASH_NOT_EXIST != ret) {
tenant_stat_ = NULL;
STORAGE_LOG(WARN, "partition map get error", KR(ret), K(tenant_id), K(lbt()));
} else {
ObSpinLockGuard guard(lock);
if (OB_SUCC(map.get_refactored(tenant_id, tenant_stat_))) {
tenant_stat_->lock_.lock();
tenant_id_ = tenant_id;
} else if (OB_HASH_NOT_EXIST != ret) {
tenant_stat_ = NULL;
STORAGE_LOG(WARN, "partition map get error", KR(ret), K(tenant_id), K(lbt()));
} else {
tenant_stat_ = object_pool.alloc();
if (OB_ISNULL(tenant_stat_)) {
ret = OB_ALLOCATE_MEMORY_FAILED;
LOG_ERROR("alloc tenant stat failed", K(ret), K(tenant_id));
} else {
tenant_stat_->tenant_id_ = tenant_id;
tenant_stat_->start_sql_renew_ts_ = ObTimeUtility::current_time();
tenant_stat_->sql_renew_count_ = 0;
if (OB_FAIL(map.set_refactored(tenant_id, tenant_stat_))) {
LOG_WARN("fail to insert and get", KR(ret), K(tenant_id));
object_pool.free(tenant_stat_);
tenant_stat_ = NULL;
} else {
tenant_stat_->lock_.lock();
tenant_id_ = tenant_id;
}
}
}
}
return ret;
}
///////////////////////////////////////////
void ObTenantSqlRenewStat::init(const uint64_t tenant_id)
{
tenant_id_ = tenant_id;
start_sql_renew_ts_ = ObTimeUtility::current_time();
sql_renew_count_ = 0;
}
void ObTenantSqlRenewStat::reset()
{
tenant_id_ = OB_INVALID_ID;
start_sql_renew_ts_ = 0;
sql_renew_count_ = 0;
}
ObLocationRpcRenewInfo::ObLocationRpcRenewInfo() : addr_(), arg_(NULL), idx_array_(NULL)
{}
ObLocationRpcRenewInfo::~ObLocationRpcRenewInfo()
{}
void ObLocationRpcRenewInfo::reset()
{
addr_.reset();
arg_ = NULL;
idx_array_ = NULL;
}
bool ObLocationRpcRenewInfo::is_valid() const
{
return addr_.is_valid() && OB_NOT_NULL(arg_) && OB_NOT_NULL(idx_array_) && arg_->keys_.count() == idx_array_->count();
}
ObReplicaRenewKey::ObReplicaRenewKey() : table_id_(OB_INVALID_ID), partition_id_(OB_INVALID_ID), addr_()
{}
ObReplicaRenewKey::ObReplicaRenewKey(const uint64_t table_id, const int64_t partition_id, const ObAddr& addr)
: table_id_(table_id), partition_id_(partition_id), addr_(addr)
{}
ObReplicaRenewKey::~ObReplicaRenewKey()
{}
uint64_t ObReplicaRenewKey::hash() const
{
uint64_t hash_val = addr_.hash();
hash_val = murmurhash(&table_id_, sizeof(table_id_), hash_val);
hash_val = murmurhash(&partition_id_, sizeof(partition_id_), hash_val);
return hash_val;
}
bool ObReplicaRenewKey::operator==(const ObReplicaRenewKey& other) const
{
return table_id_ == other.table_id_ && partition_id_ == other.partition_id_ && addr_ == other.addr_;
}
///////////////////////////////////////////
int ObLocationLeaderCache::get_strong_leader_info(const ObLocationCacheKey& key, LocationInfo& leader_info)
{
int ret = OB_SUCCESS;
int64_t idx = key.hash_v2() % CACHE_NUM;
ObLocationLeaderInfo& info = buffer_[idx];
SpinRLockGuard guard(info.get_lock());
if (OB_ISNULL(info.get_value())) {
ret = OB_ENTRY_NOT_EXIST;
} else if (key != info.get_value()->get_key()) {
ret = OB_ENTRY_NOT_EXIST;
} else {
leader_info = info.get_value()->get_strong_leader_info();
if (!leader_info.is_valid()) {
ret = OB_ENTRY_NOT_EXIST;
}
}
if (OB_FAIL(ret) && OB_ENTRY_NOT_EXIST != ret) {
LOG_WARN("fail to get leader", K(ret));
}
return ret;
}
int ObLocationLeaderCache::set_strong_leader_info(
const ObLocationCacheKey& key, const LocationInfo& location_info, bool force_update)
{
int ret = OB_SUCCESS;
int64_t idx = key.hash_v2() % CACHE_NUM;
ObLocationLeaderInfo& info = buffer_[idx];
SpinWLockGuard guard(info.get_lock());
ObLocationLeader*& value_ptr = info.get_value();
if (force_update) {
if (OB_ISNULL(value_ptr)) {
void* buf = NULL;
if (NULL == (buf = allocator_.alloc(sizeof(ObLocationLeader)))) {
ret = OB_ALLOCATE_MEMORY_FAILED;
LOG_WARN("fail to alloc mem", K(ret), K(key), K(location_info));
} else if (FALSE_IT(value_ptr = new (buf) ObLocationLeader())) {
// will not reach here
}
}
if (OB_FAIL(ret)) {
} else if (OB_ISNULL(value_ptr)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("should not be here", K(ret), K(key), K(location_info));
} else {
value_ptr->set_key(key);
value_ptr->set_strong_leader_info(location_info);
LOG_DEBUG("force set leader", K(key), K(location_info));
}
} else if (OB_NOT_NULL(value_ptr) && value_ptr->get_key() == key) {
value_ptr->set_key(key);
value_ptr->set_strong_leader_info(location_info);
LOG_DEBUG("set leader", K(key), K(location_info));
}
return ret;
}
///////////////////////////////////////////
bool ObILocationFetcher::treat_sql_as_timeout(const int error_code)
{
bool bool_ret = false;
switch (error_code) {
case OB_CONNECT_ERROR:
case OB_TIMEOUT:
case OB_WAITQUEUE_TIMEOUT:
case OB_SESSION_NOT_FOUND:
case OB_TRANS_TIMEOUT:
case OB_TRANS_STMT_TIMEOUT:
case OB_TRANS_UNKNOWN:
case OB_GET_LOCATION_TIME_OUT:
bool_ret = true;
break;
default:
bool_ret = false;
}
return bool_ret;
}
uint64_t ObILocationFetcher::get_rpc_tenant_id(const uint64_t table_id)
{
uint64_t tenant_id = 0;
if (extract_pure_id(table_id) == OB_ALL_CORE_TABLE_TID) {
tenant_id = OB_LOC_CORE_TENANT_ID;
} else if (extract_pure_id(table_id) == OB_ALL_ROOT_TABLE_TID) {
tenant_id = OB_LOC_ROOT_TENANT_ID;
} else if (is_sys_table(table_id)) {
tenant_id = OB_LOC_SYS_TENANT_ID;
} else {
tenant_id = OB_LOC_USER_TENANT_ID;
}
return tenant_id;
}
int ObILocationFetcher::partition_table_fetch_location(ObPartitionTableOperator* pt, const int64_t cluster_id,
const uint64_t table_id, const int64_t partition_id, ObPartitionLocation& location)
{
int ret = OB_SUCCESS;
ObArenaAllocator partition_info_allocator(ObModIds::OB_RS_PARTITION_TABLE_TEMP);
ObPartitionInfo partition_info;
partition_info.set_allocator(&partition_info_allocator);
bool rpc_tenant_set = false;
uint64_t old_rpc_tenant = THIS_WORKER.get_rpc_tenant();
bool need_fetch_faillist = false;
if (NULL == pt || OB_INVALID_ID == table_id || partition_id < 0) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("invalid argument", K(ret), KP(pt), K(cluster_id), KT(table_id), K(partition_id));
} else {
if (extract_pure_id(table_id) == OB_ALL_CORE_TABLE_TID) {
THIS_WORKER.set_rpc_tenant(OB_LOC_CORE_TENANT_ID);
} else if (extract_pure_id(table_id) == OB_ALL_ROOT_TABLE_TID) {
THIS_WORKER.set_rpc_tenant(OB_LOC_ROOT_TENANT_ID);
} else if (is_sys_table(table_id)) {
THIS_WORKER.set_rpc_tenant(OB_LOC_SYS_TENANT_ID);
} else {
THIS_WORKER.set_rpc_tenant(OB_LOC_USER_TENANT_ID);
}
rpc_tenant_set = true;
}
if (OB_FAIL(ret)) {
} else if (OB_FAIL(pt->get(table_id, partition_id, partition_info, need_fetch_faillist, cluster_id))) {
LOG_WARN("Partition Operator get failed", K(cluster_id), KT(table_id), K(partition_id), K(ret));
if (treat_sql_as_timeout(ret)) {
ret = OB_GET_LOCATION_TIME_OUT;
}
} else if (OB_FAIL(fill_location(partition_info, location))) {
LOG_WARN("fail to fill location", K(ret), K(partition_info));
}
if (rpc_tenant_set) {
THIS_WORKER.set_rpc_tenant(old_rpc_tenant);
}
return ret;
}
int ObILocationFetcher::partition_table_batch_fetch_location(ObPartitionTableOperator* pt, const int64_t cluster_id,
const common::ObIArray<common::ObPartitionKey>& keys, common::ObIAllocator& allocator,
common::ObIArray<ObPartitionLocation*>& new_locations)
{
int ret = OB_SUCCESS;
uint64_t old_rpc_tenant = THIS_WORKER.get_rpc_tenant();
if (keys.count() <= 0) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("keys cnt is invalid", K(ret));
} else if (NULL == pt || !keys.at(0).is_valid()) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("invalid argument", K(ret), KP(pt), "key", keys.at(0));
} else {
uint64_t table_id = keys.at(0).get_table_id();
THIS_WORKER.set_rpc_tenant(get_rpc_tenant_id(table_id));
}
ObArray<ObPartitionInfo*> partition_infos;
if (OB_FAIL(ret)) {
} else if (OB_FAIL(pt->batch_fetch_partition_infos(keys, allocator, partition_infos, cluster_id))) {
LOG_WARN("Partition Operator get failed", K(ret), K(cluster_id), "key_cnt", keys.count());
if (treat_sql_as_timeout(ret)) {
ret = OB_GET_LOCATION_TIME_OUT;
}
} else if (keys.count() != partition_infos.count()) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("cnt not match", K(ret), "key_cnt", keys.count(), "partition_cnt", partition_infos.count());
} else {
for (int64_t i = 0; OB_SUCC(ret) && i < partition_infos.count(); i++) {
ObPartitionInfo*& partition = partition_infos.at(i);
ObPartitionLocation* location = NULL;
if (OB_FAIL(ObPartitionLocation::alloc_new_location(allocator, location))) {
LOG_WARN("fail to alloc location", K(ret), KPC(partition));
} else if (OB_ISNULL(partition)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("partition is null", K(ret));
} else if (OB_FAIL(fill_location(*partition, *location))) {
LOG_WARN("fail to fill location", K(ret), KPC(partition));
} else if (OB_FAIL(new_locations.push_back(location))) {
LOG_WARN("fail to push back location", K(ret), KPC(location));
} else if (location->get_replica_locations().count() > 0) {
ObTaskController::get().allow_next_syslog();
LOG_INFO(
"location cache not hit, batch fetch location by partition_table", K(ret), K(cluster_id), KPC(location));
}
}
}
// always release partition_infos
for (int64_t i = 0; i < partition_infos.count(); i++) {
ObPartitionInfo*& partition = partition_infos.at(i);
if (OB_NOT_NULL(partition)) {
partition->~ObPartitionInfo();
partition = NULL;
}
}
if (old_rpc_tenant != THIS_WORKER.get_rpc_tenant()) {
THIS_WORKER.set_rpc_tenant(old_rpc_tenant);
}
return ret;
}
int ObILocationFetcher::fill_location(ObPartitionInfo& partition_info, ObPartitionLocation& location)
{
int ret = OB_SUCCESS;
const uint64_t table_id = partition_info.get_table_id();
const int64_t partition_id = partition_info.get_partition_id();
int64_t partition_cnt = OB_INVALID_SIZE;
ObInvalidPaxosReplicaFilter filter;
ObStatusReplicaFilter status_filter(REPLICA_STATUS_NORMAL);
LOG_TRACE(
"get partition info through partition table operator succeed", KT(table_id), K(partition_id), K(partition_info));
if (OB_FAIL(partition_info.filter(filter))) {
LOG_WARN("filter replica type failed", K(ret), K(partition_info));
} else if (OB_FAIL(partition_info.filter(status_filter))) {
LOG_WARN("fail to filter status", K(ret), K(partition_info));
} else {
location.set_table_id(table_id);
location.set_partition_id(partition_id);
const ObIArray<ObPartitionReplica>& replicas = partition_info.get_replicas_v2();
ObReplicaLocation replica_location;
for (int64_t i = 0; OB_SUCC(ret) && i < replicas.count(); ++i) {
if (!replicas.at(i).is_valid()) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("replica is not valid", "replica", replicas.at(i), K(ret));
} else {
replica_location.reset();
replica_location.server_ = replicas.at(i).server_;
replica_location.role_ = replicas.at(i).role_;
replica_location.replica_type_ = replicas.at(i).replica_type_;
replica_location.sql_port_ = replicas.at(i).sql_port_;
replica_location.property_ = replicas.at(i).property_;
replica_location.restore_status_ = replicas.at(i).is_restore_;
// TODO(): just to make it work
partition_cnt = replicas.at(i).partition_cnt_;
if (OB_FAIL(location.add(replica_location))) {
LOG_WARN("add replica location to partition location failed",
KT(table_id),
K(partition_id),
K(replica_location),
K(location),
K(ret));
}
}
}
if (OB_SUCC(ret)) {
location.set_partition_cnt(partition_cnt);
location.set_renew_time(ObTimeUtility::current_time());
location.set_sql_renew_time(ObTimeUtility::current_time());
ObReplicaLocation leader;
if (location.size() <= 0) {
ret = OB_ENTRY_NOT_EXIST;
LOG_WARN("location is empty", K(ret), K(location));
} else if (OB_FAIL(location.get_leader_by_election(leader))) {
if (OB_LOCATION_LEADER_NOT_EXIST == ret) {
ret = OB_SUCCESS;
LOG_DEBUG("location leader not exist", K(ret), K(location));
} else {
LOG_DEBUG("location get leader failed", K(ret), K(location));
}
}
}
}
if (EXECUTE_COUNT_PER_SEC(16)) {
LOG_INFO("fetch location with tenant",
"tenant",
THIS_WORKER.get_rpc_tenant(),
K(table_id),
K(partition_id),
K(ret),
K(location));
}
return ret;
}
ObLocationFetcher::ObLocationFetcher()
: inited_(false),
config_(NULL),
pt_(NULL),
remote_pt_(NULL),
rs_mgr_(NULL),
rpc_proxy_(NULL),
srv_rpc_proxy_(NULL),
locality_manager_(NULL),
cluster_id_(OB_INVALID_ID)
{}
ObLocationFetcher::~ObLocationFetcher()
{}
int ObLocationFetcher::init(ObServerConfig& config, ObPartitionTableOperator& pt,
ObRemotePartitionTableOperator& remote_pt, ObRsMgr& rs_mgr, ObCommonRpcProxy& rpc_proxy,
obrpc::ObSrvRpcProxy& srv_rpc_proxy, ObILocalityManager* locality_manager, const int64_t cluster_id)
{
int ret = OB_SUCCESS;
if (inited_) {
ret = OB_INIT_TWICE;
LOG_WARN("init twice", K(ret));
} else if (OB_ISNULL(locality_manager) || OB_INVALID_ID == cluster_id) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("invalid argument", K(ret), K(locality_manager), K(cluster_id));
} else {
config_ = &config;
pt_ = &pt;
remote_pt_ = &remote_pt;
rs_mgr_ = &rs_mgr;
rpc_proxy_ = &rpc_proxy;
srv_rpc_proxy_ = &srv_rpc_proxy;
locality_manager_ = locality_manager;
cluster_id_ = cluster_id;
inited_ = true;
}
return ret;
}
int ObLocationFetcher::fetch_location(
const uint64_t table_id, const int64_t partition_id, const int64_t cluster_id, ObPartitionLocation& location)
{
NG_TRACE(renew_loc_by_sql_begin);
int ret = OB_SUCCESS;
location.reset();
if (!inited_) {
ret = OB_NOT_INIT;
LOG_WARN("not init", K(ret));
} else if (OB_INVALID_ID == table_id || partition_id < 0) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("invalid argument", K(ret), KT(table_id), K(partition_id));
} else if (is_virtual_table(table_id)) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("table is virtual table", KT(table_id), K(ret));
} else if (cluster_id_ == cluster_id) {
int64_t tmp_cluster_id = OB_INVALID_ID;
if (OB_FAIL(partition_table_fetch_location(pt_, tmp_cluster_id, table_id, partition_id, location))) {
LOG_WARN("fetch location through partition table operator failed",
K(cluster_id),
KT(table_id),
K(partition_id),
K(ret));
} else {
ObTaskController::get().allow_next_syslog();
LOG_INFO("renew_location_with_sql success", K(table_id), K(partition_id), K(location));
}
} else {
if (OB_FAIL(partition_table_fetch_location(remote_pt_, cluster_id, table_id, partition_id, location))) {
LOG_WARN("fetch location through remote partition table operator failed",
K(cluster_id_),
K(cluster_id),
KT(table_id),
K(partition_id),
K(ret));
} else {
ObTaskController::get().allow_next_syslog();
LOG_INFO("renew_remote_location_with_sql success",
K(table_id),
K(partition_id),
K(cluster_id_),
K(cluster_id),
K(location));
}
}
NG_TRACE_EXT(renew_loc_by_sql_end, OB_ID(ret), ret, OB_ID(table_id), table_id, OB_ID(partition_id), partition_id);
return ret;
}
int ObLocationFetcher::fetch_vtable_location(const uint64_t table_id, ObSArray<ObPartitionLocation>& locations)
{
NG_TRACE(fetch_vtable_loc_begin);
int ret = OB_SUCCESS;
if (!inited_) {
ret = OB_NOT_INIT;
LOG_WARN("not init", K(ret));
} else if (OB_INVALID_ID == table_id || !is_virtual_table(table_id)) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("table is not virtual table", KT(table_id), K(ret));
} else {
LOG_INFO("fetch virtual table location with tenant", "tenant", THIS_WORKER.get_rpc_tenant(), KT(table_id));
int64_t timeout_us = GCONF.location_cache_refresh_sql_timeout;
if (ObTimeoutCtx::get_ctx().is_timeout_set()) {
timeout_us = std::min(timeout_us, ObTimeoutCtx::get_ctx().get_timeout());
}
if (timeout_us <= 0) {
ret = OB_TIMEOUT;
LOG_WARN("process timeout", K(ret), K(timeout_us));
} else if (OB_ISNULL(rs_mgr_)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("rs_mgr is null", KR(ret));
} else if (OB_FAIL(rpc_proxy_->to_rs(*rs_mgr_)
.by(OB_LOC_SYS_TENANT_ID)
.timeout(timeout_us)
.fetch_location(table_id, locations))) {
LOG_WARN("fetch location through rpc failed", KT(table_id), K(ret));
} else {
const int64_t now = ObTimeUtility::current_time();
for (int64_t i = 0; OB_SUCC(ret) && i < locations.count(); ++i) {
locations.at(i).set_renew_time(now);
}
if (0 == locations.count()) {
ret = OB_ENTRY_NOT_EXIST;
LOG_WARN("locations is empty", K(ret));
}
}
}
NG_TRACE_EXT(fetch_vtable_loc_end, OB_ID(ret), ret, OB_ID(table_id), table_id);
return ret;
}
int ObLocationFetcher::batch_fetch_location(const common::ObIArray<common::ObPartitionKey>& keys,
const int64_t cluster_id, common::ObIAllocator& allocator, common::ObIArray<ObPartitionLocation*>& new_locations)
{
int ret = OB_SUCCESS;
if (!inited_) {
ret = OB_NOT_INIT;
LOG_WARN("not init", K(ret));
} else if (keys.count() <= 0) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("keys count is invalid", K(ret));
} else if (cluster_id_ == cluster_id) {
int64_t tmp_cluster_id = OB_INVALID_ID;
if (OB_FAIL(partition_table_batch_fetch_location(pt_, tmp_cluster_id, keys, allocator, new_locations))) {
LOG_WARN("fetch location through partition table operator failed", K(ret), K(cluster_id), "cnt", keys.count());
} else {
LOG_INFO("batch_renew_location_with_sql success", K(ret), "cnt", keys.count());
}
} else {
if (OB_FAIL(partition_table_batch_fetch_location(remote_pt_, cluster_id, keys, allocator, new_locations))) {
LOG_WARN("fetch location through remote partition table operator failed",
K(ret),
K(cluster_id_),
K(cluster_id),
"cnt",
keys.count());
} else {
LOG_INFO(
"batch_renew_remote_location_with_sql success", K(ret), K(cluster_id_), K(cluster_id), "cnt", keys.count());
}
}
return ret;
}
/*
* In most cases, location cache of sys tenant are almost the same. So, we can try to use __all_core_table's
* location_cache to renew other system tables' location cache in sys tenant, which may also reduce the possibility
* of circular dependency.
*/
int ObLocationFetcher::batch_renew_sys_table_location_by_rpc(const ObPartitionLocation& core_table_location,
const common::ObIArray<ObLocationCacheKey>& keys, common::ObIArray<ObLocationLeader>& results)
{
int ret = OB_SUCCESS;
if (!inited_) {
ret = OB_NOT_INIT;
LOG_WARN("not init", K(ret));
} else if (OB_ISNULL(srv_rpc_proxy_)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("srv_rpc_proxy is null", K(ret));
} else {
results.reset();
obrpc::ObBatchGetRoleArg arg;
const int64_t timeout_ts = GCONF.location_cache_refresh_rpc_timeout;
rootserver::ObBatchGetRoleProxy proxy(*srv_rpc_proxy_, &obrpc::ObSrvRpcProxy::batch_get_role);
const ObIArray<ObReplicaLocation>& location_array = core_table_location.get_replica_locations();
/*
* partition_cnt is meaningful to construct ObPartitionKey before ver 1.4.40,
* and its value is 0 if table is created not less than ver 1.4.40.
*
* For compatibility, we should get partition_cnt from schema service, storage, or meta table.
* But this function only works when partition's location cache is empty, which means we can get partiton_cnt from
* nowhere.
*
* Because we won't upgrade cluster from ver 1.4.x to ver 2.x/3.x any more (Use OMS to migrate data across clusters
* instead), we pretend partition_cnt is 0 here.
*/
for (int64_t i = 0; OB_SUCC(ret) && i < keys.count(); i++) {
ObPartitionKey key(keys.at(i).table_id_, keys.at(i).partition_id_, 0);
if (OB_FAIL(arg.keys_.push_back(key))) {
LOG_WARN("fail to push back arg", K(ret), K(key));
}
}
for (int64_t i = 0; i < location_array.count() && OB_SUCC(ret); i++) {
const ObAddr& addr = location_array.at(i).server_;
if (common::REPLICA_TYPE_FULL != location_array.at(i).replica_type_) {
// skip
} else if (OB_FAIL(proxy.call(addr, timeout_ts, arg))) {
LOG_WARN("fail to get role", K(ret), K(addr), K(arg));
}
}
ObArray<int> return_code_array;
int tmp_ret = OB_SUCCESS; // always wait all
if (OB_SUCCESS != (tmp_ret = proxy.wait_all(return_code_array))) {
LOG_WARN("wait batch result failed", K(tmp_ret), K(ret));
ret = OB_SUCC(ret) ? tmp_ret : ret;
}
for (int64_t i = 0; OB_SUCC(ret) && i < return_code_array.count(); i++) {
int res_ret = return_code_array.at(i);
if (OB_SUCCESS == res_ret) {
const ObBatchGetRoleResult* result = proxy.get_results().at(i);
if (OB_ISNULL(result)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("result is null", K(ret));
} else {
const ObBatchGetRoleResult* result = proxy.get_results().at(i);
if (OB_ISNULL(result)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("result is null", K(ret));
} else if (result->results_.count() != arg.keys_.count()) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("count not match", K(ret), "keys_cnt", arg.keys_.count(), "result_cnt", result->results_.count());
} else {
const int64_t now = common::ObTimeUtility::current_time();
for (int64_t j = 0; OB_SUCC(ret) && j < result->results_.count(); j++) {
if (OB_SUCCESS == result->results_.at(j)) {
ObLocationLeader info;
info.set_strong_leader_info(LocationInfo(proxy.get_dests().at(i), now));
info.set_key(keys.at(j));
if (OB_FAIL(results.push_back(info))) {
LOG_WARN("fail to push back result", K(ret), K(info));
}
}
}
}
}
}
ret = OB_SUCCESS; // overwrite ret;
}
}
return ret;
}
/*
* Consider tenant locality may across multi region, it's a high cost if we fetch location cache
* by sending rpc to all servers in member_list. So this function would only send rpc
* to server which is in member_list and local region. In some cases, non paxos members' location cache
* may be imprecise if we only renew location cache by RPC.
*
* TODO: () renew location asynchronously if only non-paxos member has changed.
*/
int ObLocationFetcher::renew_location_with_rpc_v2(
common::hash::ObHashMap<ObReplicaRenewKey, const ObMemberListAndLeaderArg*>* result_map,
const ObPartitionLocation& cached_location, ObPartitionLocation& new_location, bool& is_new_location_valid)
{
NG_TRACE(renew_loc_by_rpc_begin);
int ret = OB_SUCCESS;
is_new_location_valid = true;
common::ObPartitionKey pkey;
const uint64_t tenant_id = get_rpc_tenant_id(cached_location.get_table_id());
const int64_t default_timeout = GCONF.location_cache_refresh_rpc_timeout;
int64_t timeout_us = default_timeout;
const ObIArray<ObReplicaLocation>& location_array = cached_location.get_replica_locations();
ObMemberListAndLeaderArg member_info;
const ObMemberListAndLeaderArg* member_info_ptr = &member_info;
ObReplicaLocation old_leader;
ObReplicaLocation new_leader; // only use server_ and role_
ObSEArray<ObReplicaMember, OB_DEFAULT_SE_ARRAY_COUNT> non_paxos_replicas;
bool batch_renew = OB_NOT_NULL(result_map);
// can get restore leader by RPC not less than ver 2.2.74
bool new_mode = GET_MIN_CLUSTER_VERSION() >= CLUSTER_VERSION_2274;
if (!inited_) {
ret = OB_NOT_INIT;
LOG_WARN("not inited yet", KR(ret));
} else if (!cached_location.is_valid()) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("invalid args", KR(ret), K(cached_location));
} else if (OB_FAIL(cached_location.get_partition_key(pkey))) {
LOG_WARN("failed to get partition key", KR(ret), K(cached_location));
} else if (OB_FAIL(new_location.assign(cached_location))) {
LOG_WARN("failed to assign location", KR(ret), K(cached_location));
} else {
ret = new_mode ? cached_location.get_leader_by_election(old_leader) : cached_location.get_strong_leader(old_leader);
if (OB_LOCATION_LEADER_NOT_EXIST == ret) {
ret = OB_SUCCESS;
LOG_DEBUG("no leader in cached location", K(new_mode), K(cached_location));
} else if (OB_FAIL(ret)) {
LOG_DEBUG("failed to get leader", KR(ret), K(new_mode), K(cached_location));
}
}
/*
* Using infos by RPC from servers in member_list(from old location cache)
* and in local region to construct new location cache.
*
* If there are only the following difference between two location cache,
* we consider renew location by RPC successfully.
* 1. only leader changed.
* 2. member_list not changed.
* 3. only paxos member's replica type changed(F->L).
* 4. non paxos member not changed.
*/
int64_t local_paxos_cnt = 0;
ObSEArray<ObAddr, OB_DEFAULT_SE_ARRAY_COUNT> leader_infos; // leader list gather from server in local region
for (int64_t i = 0; OB_SUCC(ret) && is_new_location_valid && i < location_array.count(); i++) {
const ObReplicaLocation& replica_location = location_array.at(i);
const ObAddr& addr = replica_location.server_;
bool is_local = false;
if (ObTimeoutCtx::get_ctx().is_timeout_set()) {
timeout_us = std::min(ObTimeoutCtx::get_ctx().get_timeout(), default_timeout);
}
if (timeout_us <= 0) {
ret = OB_GET_LOCATION_TIME_OUT;
LOG_WARN("get location operation is already timeout", KR(ret), K(timeout_us));
} else if (OB_ISNULL(locality_manager_)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("locality manager ptr is null", KR(ret));
} else if (OB_FAIL(locality_manager_->is_local_server(addr, is_local))) {
LOG_WARN("fail to check is local server", KR(ret), K(addr));
} else if (!is_local || !ObReplicaTypeCheck::is_paxos_replica_V2(replica_location.replica_type_)) {
continue;
} else if (FALSE_IT(local_paxos_cnt++)) {
} else if (!batch_renew) {
member_info.reset();
/*
* Old implement of get_member_list_and_leader () will cause error when we call on restore replica.
* When old and new observers are mixed in upgrade, renew location by RPC in new observer will cause error
* and we will renew location by SQL instead.
*/
if (OB_ISNULL(srv_rpc_proxy_)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("srv_rpc_proxy ptr is null", KR(ret));
} else if (OB_FAIL(srv_rpc_proxy_->to(addr)
.by(tenant_id)
.timeout(timeout_us)
.get_member_list_and_leader(pkey, member_info))) {
LOG_WARN("fail to get replica info", KR(ret), K(addr), K(pkey), K(timeout_us), K(old_leader), K(member_info));
} else {
member_info_ptr = &member_info;
}
} else {
ObReplicaRenewKey key(pkey.get_table_id(), pkey.get_partition_id(), addr);
if (OB_FAIL(result_map->get_refactored(key, member_info_ptr))) {
LOG_WARN("fail to get member info from map", KR(ret), K(key));
}
}
if (OB_FAIL(ret)) {
} else if (OB_ISNULL(member_info_ptr)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("member_info is null", KR(ret), K(pkey));
} else {
LOG_DEBUG("get leader_and_member_list", K(pkey), K(addr), KPC(member_info_ptr));
if (OB_FAIL(deal_with_replica_type_changed(
new_mode, pkey, replica_location, *member_info_ptr, new_location, is_new_location_valid))) {
LOG_WARN("fail to deal with replica_type changed",
KR(ret),
K(new_mode),
K(pkey),
K(replica_location),
KPC(member_info_ptr));
} else if (!is_new_location_valid) {
// skip
} else if (OB_FAIL(add_non_paxos_replica(*member_info_ptr, non_paxos_replicas))) {
LOG_WARN("fail to add readonly replica", KR(ret), K(new_mode), K(pkey), KPC(member_info_ptr));
} else if (OB_FAIL(check_leader_and_member_list(
new_mode, pkey, addr, location_array, *member_info_ptr, new_leader, is_new_location_valid))) {
LOG_WARN("fail to check leader and member_list",
KR(ret),
K(new_mode),
K(pkey),
K(addr),
K(location_array),
KPC(member_info_ptr));
} else if (!new_leader.is_valid() && member_info_ptr->leader_.is_valid() &&
!has_exist_in_array(leader_infos, member_info_ptr->leader_) &&
OB_FAIL(leader_infos.push_back(member_info_ptr->leader_))) {
// Use leader returned by follower if local region don't have leader.
LOG_WARN("fail to push back leader", KR(ret), KPC(member_info_ptr));
}
}
} // end for
if (local_paxos_cnt <= 0) {
// double check, local region should have at least one paxos member if we come here.
is_new_location_valid = false;
ObTaskController::get().allow_next_syslog();
LOG_INFO("local region paxos member cnt less than 1, rpc renew should failed",
"table_id",
cached_location.get_table_id(),
"partition_id",
cached_location.get_partition_id(),
K(cached_location),
K(lbt()));
}
if (OB_SUCC(ret) && !new_leader.server_.is_valid() && is_new_location_valid) {
// no leader in local region, try use leader from followers(avoid renew location cache by SQL)
if (1 == leader_infos.count()) {
if (!GCTX.is_standby_cluster() || ObMultiClusterUtil::is_cluster_private_table(cached_location.get_table_id())) {
/*
* If local region have no leader, we can use followers' leader instead.
*
* In Primary Cluster:
* 1. While locality across multi regions:
* At last followers will get lastest leader by election.
* 2. Physical restore (create partitions with member_list):
* Followers can't get leader's role(leader maybe is restore leader), we pretend leader from
* followers is strong leader because we won't(or can't) strong read or write from restore leader.
*
* In Standby Cluster:
* For cluster private table, leader must be strong leader.
*
* So, we can pretend leader is strong leader while we can only get leader from followers in local region.
*/
new_leader.server_ = leader_infos.at(0);
new_leader.role_ = LEADER;
} else {
/*
* Normally, standby cluster won't across regions. For non cluster private table in standby cluster,
* we renew location by SQL if there are no leader in local region.
*/
is_new_location_valid = false;
}
}
}
// TODO:() When leader not exist, renew location asynchronously.
// check if non paxos members are not changed.
if (OB_SUCC(ret) && is_new_location_valid) {
bool is_same = false;
if (OB_FAIL(check_non_paxos_replica(location_array, non_paxos_replicas, is_same))) {
LOG_WARN("fail to check readonly replica", KR(ret), K(new_mode), K(cached_location), K(non_paxos_replicas));
} else if (!is_same) {
is_new_location_valid = false;
ObTaskController::get().allow_next_syslog();
LOG_INFO("check non paxos replica not same, need renew by sql",
K(pkey),
K(new_mode),
K(location_array),
K(non_paxos_replicas));
} else if (OB_FAIL(new_location.change_leader(new_leader))) {
LOG_WARN("failed to change leader", KR(ret), K(new_mode), K(new_leader), K(new_location), K(cached_location));
} else {
new_location.set_renew_time(ObTimeUtility::current_time());
new_location.unmark_fail();
if (old_leader.server_ != new_leader.server_) {
ObTaskController::get().allow_next_syslog();
LOG_INFO("leader changed, renew_location_with_rpc success",
"table_id",
cached_location.get_table_id(),
"partition_id",
cached_location.get_partition_id(),
K(new_mode),
K(cached_location),
K(new_location),
K(lbt()));
} else {
LOG_DEBUG("renew_location_with_rpc success",
"table_id",
cached_location.get_table_id(),
"partition_id",
cached_location.get_partition_id(),
K(new_mode),
K(cached_location),
K(new_location),
K(lbt()));
}
}
}
if (OB_FAIL(ret) || !is_new_location_valid) {
new_location.reset();
is_new_location_valid = false;
}
EVENT_INC(LOCATION_CACHE_RPC_CHECK);
NG_TRACE_EXT(renew_loc_by_rpc_end,
OB_ID(ret),
ret,
OB_ID(table_id),
pkey.get_table_id(),
OB_ID(partition_id),
pkey.get_partition_id());
return ret;
}
/*
* @description:
* Check if only replica type of paxos member is changed.
* If so, update location cache and renew location cache by RPC successfully.
* Otherwise, try update location cache by SQL later.
* @param[in]:
* - new_mode : new_mode = true means role is meaningful
* - pkey : partition key
* - old_replica_location : old location cache
* - member_info : got from rpc
* @param[out]:
* - new_location : new location cache
* @param[in/out]:
* - is_new_location_valid: If it's true, it means renew location by RPC is successful for now.
* @return err_code
*/
int ObLocationFetcher::deal_with_replica_type_changed(const bool new_mode, const ObPartitionKey& pkey,
const ObReplicaLocation& old_replica_location, const ObMemberListAndLeaderArg& member_info,
ObPartitionLocation& new_location, bool& is_new_location_valid)
{
int ret = OB_SUCCESS;
const ObAddr& addr = old_replica_location.server_;
// check if paxos member's replica type is changed by replica type from rpc
if (!is_new_location_valid) {
// skip
} else if (GET_MIN_CLUSTER_VERSION() < CLUSTER_VERSION_1460) {
// skip
} else if (common::REPLICA_TYPE_MAX == member_info.replica_type_) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("replica type is invalid", KR(ret), K(pkey), K(member_info));
} else if (new_mode && common::INVALID_ROLE == member_info.role_) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("role should be valid", KR(ret), K(member_info));
} else if (!member_info.check_leader_is_valid()) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("member_info's self_/leader_/role_ not match", KR(ret), K(new_mode), K(pkey), K(member_info));
} else if (common::REPLICA_TYPE_FULL != member_info.replica_type_ && addr == member_info.leader_) {
// defence, leader's replica type must be F.
ret = OB_ERR_UNEXPECTED;
LOG_WARN("leader replica_type should be F", KR(ret), K(pkey), K(member_info), K(addr));
} else if (!ObReplicaTypeCheck::is_paxos_replica_V2(member_info.replica_type_)) {
// If paxos member's replica type change to R, it should renew location cache by SQL,
// because only renew location cache by RPC can't get precise non paxos member's locations.
is_new_location_valid = false;
ObTaskController::get().allow_next_syslog();
LOG_INFO("member replica_type has changed, need renew by sql", K(pkey), K(member_info));
} else if (old_replica_location.replica_type_ != member_info.replica_type_) {
// replica type changed, but it's still paxos type.
ObReplicaLocation replica_location = old_replica_location;
replica_location.replica_type_ = member_info.replica_type_;
if (addr == member_info.leader_) { // new_mode = true means role is meaningful
replica_location.role_ = new_mode ? member_info.role_ : LEADER;
} else {
replica_location.role_ = FOLLOWER;
}
replica_location.property_ = member_info.property_;
if (OB_FAIL(new_location.update(addr, replica_location))) {
LOG_WARN("update cached_location fail", KR(ret), K(pkey), K(addr), K(replica_location));
} else {
ObTaskController::get().allow_next_syslog();
LOG_INFO("member replica_type has changed, need modify", K(pkey), K(new_location), K(member_info));
}
}
return ret;
}
/*
* @description:
* check if leader and member_list are same with old location.
* 1. If leader exist, check leader's member_list only.
* 2. If leader doesn't exist, check all followers' member_list.
* @param[in]:
* - new_mode : new_mode = true means role is meaningful
* - pkey : partition key
* - addr : rpc destination
* - location_array : old location cache
* - member_info : got from rpc
* @param[in/out]:
* - new_leader : leader got by RPC
* - is_new_location_valid: If it's true, it means renew location by RPC is successful for now.
* @return err_code
*/
int ObLocationFetcher::check_leader_and_member_list(const bool new_mode, const ObPartitionKey& pkey, const ObAddr& addr,
const ObIArray<ObReplicaLocation>& location_array, const ObMemberListAndLeaderArg& member_info,
ObReplicaLocation& new_leader, bool& is_new_location_valid)
{
int ret = OB_SUCCESS;
if (!is_new_location_valid) {
// skip
} else if (new_mode && common::INVALID_ROLE == member_info.role_) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("role should be valid", KR(ret), K(member_info));
} else if (!member_info.check_leader_is_valid()) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("member_info's self_/leader_/role_ not match", KR(ret), K(new_mode), K(pkey), K(member_info));
} else if (new_leader.server_.is_valid()) {
if (member_info.leader_.is_valid() && member_info.leader_ != new_leader.server_) {
// If we can different leader by RPC, it means we need renew location by SQL.
is_new_location_valid = false;
ObTaskController::get().allow_next_syslog();
LOG_WARN("has multi leaders, rpc renew failed", K(pkey), K(new_leader), K(member_info));
} else {
/*
* new_leader.server_.is_valid() means we got leader once
* and leader's member_list is same with old location cache.
* So, we do nothing in this branch.
*/
}
} else if (OB_FAIL(check_member_list(location_array, member_info.member_list_, is_new_location_valid))) {
LOG_WARN("fail to check member list", KR(ret), K(pkey), K(location_array), K(member_info));
} else if (!is_new_location_valid) {
ObTaskController::get().allow_next_syslog();
LOG_WARN("obs 'member_list is not same with cached location, rpc renew failed",
K(pkey),
K(member_info),
K(location_array));
} else if (addr == member_info.leader_) {
new_leader.role_ = new_mode ? member_info.role_ : LEADER;
new_leader.server_ = addr;
}
return ret;
}
// add non paxos member got by RPC to array
int ObLocationFetcher::add_non_paxos_replica(
const ObMemberListAndLeaderArg& member_info, ObIArray<ObReplicaMember>& non_paxos_replicas)
{
int ret = OB_SUCCESS;
bool is_local = false;
const ObIArray<ObReplicaMember>& lower_list = member_info.lower_list_;
const common::ObIArray<common::ObAddr>& member_list = member_info.member_list_;
if (!inited_) {
ret = OB_NOT_INIT;
LOG_WARN("not inited yet", KR(ret));
}
for (int64_t i = 0; i < lower_list.count() && OB_SUCC(ret); i++) {
ObReplicaMember replica = lower_list.at(i);
bool finded = false;
for (int64_t j = 0; j < member_list.count() && !finded; j++) {
if (replica.get_server() == member_list.at(j)) {
finded = true;
}
}
if (finded) {
// {non paxos members} = {lower_list} - {member_list}
} else if (OB_ISNULL(locality_manager_)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("locality manager ptr is null", KR(ret));
} else if (OB_FAIL(locality_manager_->is_local_server(replica.get_server(), is_local))) {
if (OB_ENTRY_NOT_EXIST != ret) {
LOG_WARN("check if server in local region fail", K(ret), K(replica));
} else {
LOG_INFO("fail to find serer", "server", replica.get_server());
// nothing todo
}
} else if (!is_local) {
// filter non paxos member in other region
LOG_DEBUG("replica is not local, ignore", K(replica));
} else if (has_exist_in_array(non_paxos_replicas, replica)) {
LOG_WARN("invalid lower list, should not same with other", K(non_paxos_replicas), K(lower_list));
} else if (OB_FAIL(non_paxos_replicas.push_back(replica))) {
LOG_WARN("fail to push back", K(ret), K(non_paxos_replicas));
}
}
return ret;
}
// check if non paxos members are not changed
int ObLocationFetcher::check_non_paxos_replica(const common::ObIArray<ObReplicaLocation>& cached_member_list,
const common::ObIArray<ObReplicaMember>& non_paxos_replicas, bool& is_same)
{
int ret = OB_SUCCESS;
is_same = true;
int64_t non_paxos_relica_num = 0;
bool is_local = false;
if (!inited_) {
ret = OB_NOT_INIT;
LOG_WARN("not inited yet", KR(ret));
}
// only check non paxos member from old location cache in local region
for (int64_t i = 0; OB_SUCC(ret) && i < cached_member_list.count() && is_same; i++) {
const ObReplicaLocation& location = cached_member_list.at(i);
if (ObReplicaTypeCheck::is_paxos_replica_V2(location.replica_type_)) {
// nothing todo
} else if (OB_ISNULL(locality_manager_)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("locality manager ptr is null", KR(ret));
} else if (OB_FAIL(locality_manager_->is_local_server(location.server_, is_local))) {
LOG_DEBUG("fail to check is local server", K(ret), K(location));
ret = OB_SUCCESS;
is_same = false;
} else if (!is_local) {
// nothing todo
} else {
non_paxos_relica_num++;
bool found = false;
for (int64_t j = 0; OB_SUCC(ret) && is_same && j < non_paxos_replicas.count(); j++) {
if (non_paxos_replicas.at(j).get_server() == location.server_) {
found = true;
}
} // end for
if (!found) {
LOG_DEBUG("fail to find server, not same", K(location.server_), K(non_paxos_replicas));
is_same = false;
}
}
} // end for
if (OB_SUCC(ret) && is_same) {
if (non_paxos_relica_num != non_paxos_replicas.count()) {
LOG_DEBUG("non paxos replica num is not equal ; not same", K(non_paxos_relica_num), K(non_paxos_replicas));
is_same = false;
}
}
return ret;
}
int ObLocationFetcher::check_member_list(const common::ObIArray<ObReplicaLocation>& cached_member_list,
const common::ObIArray<common::ObAddr>& member_list, bool& is_same)
{
int ret = OB_SUCCESS;
bool found = false;
is_same = true;
int64_t member_cnt = 0;
for (int64_t i = 0; is_same && i < cached_member_list.count(); ++i) {
found = false;
if (!ObReplicaTypeCheck::is_paxos_replica_V2(cached_member_list.at(i).replica_type_)) {
// nothing todo
} else {
member_cnt++;
for (int64_t j = 0; !found && j < member_list.count(); ++j) {
if (member_list.at(j) == cached_member_list.at(i).server_) {
found = true;
}
}
if (!found) {
is_same = false;
}
}
}
if (OB_SUCC(ret) && is_same) {
if (member_cnt != member_list.count()) {
is_same = false;
}
}
return ret;
}
int ObLocationFetcher::batch_renew_location_with_rpc(const common::ObIArray<const ObPartitionLocation*>& rpc_locations,
common::ObIAllocator& allocator, common::ObIArray<ObPartitionLocation*>& new_locations)
{
int ret = OB_SUCCESS;
common::ObArenaAllocator arg_allocator("RenewLocation");
common::ObArray<ObLocationRpcRenewInfo> infos;
common::ObArray<int> key_results; // key_results corresponds to rpc_locations
common::hash::ObHashMap<ObReplicaRenewKey, const ObMemberListAndLeaderArg*> result_map;
int64_t now = ObTimeUtility::current_time();
if (!inited_) {
ret = OB_NOT_INIT;
LOG_WARN("not init", K(ret));
} else if (rpc_locations.count() <= 0) {
// do nothing
} else if (OB_FAIL(init_batch_rpc_renew_struct(rpc_locations, arg_allocator, infos, key_results, result_map))) {
LOG_WARN("fail to init struct", K(ret));
} else if (OB_ISNULL(srv_rpc_proxy_)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("srv_rpc_proxy is null", K(ret));
} else {
// We should renew location by SQL in the following cases:
// case 1. Call RPC on observer failed.
// case 2. Partial partitions' in one RPC return failure.
// case 3. Partial partitions' new locations not matched.(member_list or non paxos member maybe changed)
rootserver::ObBatchRpcRenewLocProxy proxy_batch(
*srv_rpc_proxy_, &obrpc::ObSrvRpcProxy::batch_get_member_list_and_leader);
// gather rpc results
const int64_t timeout_ts = GCONF.location_cache_refresh_rpc_timeout;
for (int64_t i = 0; i < infos.count(); i++) { // ignore failure
ObLocationRpcRenewInfo& info = infos.at(i);
if (!info.is_valid()) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("invalid info", K(ret), K(info));
} else {
ObAddr& addr = info.addr_;
ObLocationRpcRenewArg& arg = *(info.arg_);
const uint64_t tenant_id = get_rpc_tenant_id(arg.keys_.at(0).get_table_id());
if (OB_FAIL(proxy_batch.call(addr, timeout_ts, tenant_id, arg))) {
LOG_WARN("fail to call async batch rpc", K(ret), K(addr), K(arg), K(tenant_id));
}
}
ret = OB_SUCCESS;
}
ObArray<int> return_code_array;
int tmp_ret = OB_SUCCESS; // always wait all
if (OB_SUCCESS != (tmp_ret = proxy_batch.wait_all(return_code_array))) {
LOG_WARN("wait batch result failed", K(tmp_ret), K(ret));
ret = OB_SUCC(ret) ? tmp_ret : ret;
}
if (OB_FAIL(ret)) {
} else if (return_code_array.count() != infos.count() ||
return_code_array.count() != proxy_batch.get_results().count()) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("cnt not match",
K(ret),
"return_cnt",
return_code_array.count(),
"result_cnt",
proxy_batch.get_results().count(),
"arg_cnt",
infos.count());
}
for (int64_t i = 0; OB_SUCC(ret) && i < return_code_array.count(); i++) {
int res_ret = return_code_array.at(i);
ObAddr& addr = infos.at(i).addr_;
ObLocationRpcRenewArg*& arg = infos.at(i).arg_;
ObArray<int64_t>*& idx_array = infos.at(i).idx_array_;
if (!infos.at(i).is_valid()) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("renew info is invalid", K(ret), "info", infos.at(i));
} else if (OB_SUCCESS != res_ret) { // case 1
for (int j = 0; OB_SUCC(ret) && j < idx_array->count(); j++) {
ObPartitionKey& pkey = arg->keys_.at(j);
int64_t idx = idx_array->at(j);
int& result = key_results.at(idx);
if (OB_SUCCESS == result) {
result = res_ret;
LOG_INFO("rcode is not success, rpc renew failed", K(result), K(addr), K(pkey));
}
}
} else {
const ObLocationRpcRenewResult* result = proxy_batch.get_results().at(i);
if (OB_ISNULL(result)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("result is null", K(ret), KP(result));
} else if (result->results_.count() != arg->keys_.count()) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("cnt not match", K(ret), "result_cnt", result->results_.count(), "arg_cnt", arg->keys_.count());
}
for (int64_t j = 0; OB_SUCC(ret) && j < result->results_.count(); j++) {
const ObMemberListAndLeaderArg& res = result->results_.at(j);
ObPartitionKey& pkey = arg->keys_.at(j);
if (!res.is_valid()) { // case 2
int64_t idx = idx_array->at(j);
int& result = key_results.at(idx);
if (OB_SUCCESS == result) {
result = OB_RENEW_LOCATION_BY_RPC_FAILED;
LOG_INFO("result is not success, rpc renew failed", K(result), K(addr), K(pkey));
}
} else {
ObReplicaRenewKey key(pkey.get_table_id(), pkey.get_partition_id(), addr);
if (OB_FAIL(result_map.set_refactored(key, &res))) {
LOG_WARN("fail to set value", K(ret), K(key));
}
}
}
}
}
// renew location
for (int64_t i = 0; i < key_results.count(); i++) { // ignore failure
bool is_new_location_valid = false;
if (OB_SUCCESS != key_results.at(i)) {
ret = key_results.at(i);
} else if (OB_ISNULL(rpc_locations.at(i))) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("location is null", K(ret), K(i));
} else {
const ObPartitionLocation* const& location = rpc_locations.at(i);
ObPartitionLocation* new_location = NULL;
if (OB_FAIL(ObPartitionLocation::alloc_new_location(allocator, new_location))) {
LOG_WARN("fail to alloc new location", K(ret), KPC(location));
} else if (OB_FAIL(renew_location_with_rpc_v2(&result_map, *location, *new_location, is_new_location_valid))) {
LOG_WARN(
"failed to renew location with rpc", K(ret), KPC(location), "cost", ObTimeUtility::current_time() - now);
} else if (!is_new_location_valid) { // case 3
// skip
} else if (OB_FAIL(new_locations.push_back(new_location))) {
LOG_WARN("fail to push back location", K(ret), KPC(new_location));
} else if (EXECUTE_COUNT_PER_SEC(16)) {
LOG_INFO("fetch location with rpc success",
KPC(location),
KPC(new_location),
"cost",
ObTimeUtility::current_time() - now);
}
}
if (OB_FAIL(ret) || !is_new_location_valid) {
EVENT_INC(LOCATION_CACHE_RPC_RENEW_FAIL);
LOG_WARN("rpc renew failed, need sql renew",
K(ret),
"old_location",
*(rpc_locations.at(i)),
"cost",
ObTimeUtility::current_time() - now);
}
ret = OB_SUCCESS;
}
}
return ret;
}
/*
* rpc_locations : Partitions that we should try to renew their locations by RPC.
* infos : organized by server, record pkey and its corresponding index of rpc_locations.
* key_results : rpc renew results corresponding to rpc_locations.
* result_map : HashMap. Key is tuple (table_id, partition_id, addr). Value is rpc result of partition on single server.
*/
int ObLocationFetcher::init_batch_rpc_renew_struct(const common::ObIArray<const ObPartitionLocation*>& rpc_locations,
common::ObIAllocator& allocator, common::ObArray<ObLocationRpcRenewInfo>& infos, common::ObArray<int>& key_results,
common::hash::ObHashMap<ObReplicaRenewKey, const ObMemberListAndLeaderArg*>& result_map)
{
int ret = OB_SUCCESS;
if (!inited_) {
ret = OB_NOT_INIT;
LOG_WARN("not inited yet", KR(ret));
} else if (rpc_locations.count() <= 0) {
// do nothing
} else {
ObLocationRpcRenewInfo info;
common::hash::ObHashMap<ObAddr, int64_t> dest_map; // record index of infos
int64_t bucket_num = UNIQ_TASK_QUEUE_BATCH_EXECUTE_NUM * OB_MAX_MEMBER_NUMBER;
if (OB_FAIL(dest_map.create(bucket_num, "LocRpcFetcher", "LocRpcFetcher"))) {
LOG_WARN("fail to init dest_map", K(ret));
} else if (OB_FAIL(result_map.create(bucket_num, "LocRpcResMap", "LocRpcResMap"))) {
LOG_WARN("fail to init result_map ", K(ret));
}
for (int64_t i = 0; OB_SUCC(ret) && i < rpc_locations.count(); i++) {
info.reset();
const ObPartitionLocation* const& location = rpc_locations.at(i);
bool renew_by_sql = true;
if (OB_ISNULL(location)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("location is null", K(ret));
} else {
const uint64_t table_id = location->get_table_id();
const int64_t partition_id = location->get_partition_id();
ObPartitionKey key(location->get_table_id(), location->get_partition_id(), 0);
const ObIArray<ObReplicaLocation>& location_array = location->get_replica_locations();
for (int64_t j = 0; OB_SUCC(ret) && j < location_array.count(); j++) {
const ObAddr& addr = location_array.at(j).server_;
int64_t idx = OB_INVALID_INDEX;
bool is_local = false;
if (OB_ISNULL(locality_manager_)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("locality manager ptr is null", KR(ret));
} else if (OB_FAIL(locality_manager_->is_local_server(addr, is_local))) {
LOG_WARN("fail to check is local server", K(ret), K(addr));
} else if (!is_local || !ObReplicaTypeCheck::is_paxos_replica_V2(location_array.at(j).replica_type_)) {
// nothing todo
} else if (OB_FAIL(dest_map.get_refactored(addr, idx))) {
if (OB_HASH_NOT_EXIST != ret) {
LOG_WARN("fail to get value", K(ret), K(addr));
} else {
// init ObLocationRpcRenewInfo
void* buf = NULL;
if (OB_ISNULL(buf = allocator.alloc(sizeof(ObLocationRpcRenewArg)))) {
ret = OB_ALLOCATE_MEMORY_FAILED;
LOG_WARN("fail to alloc mem", K(ret));
} else if (FALSE_IT(info.arg_ = new (buf) ObLocationRpcRenewArg(allocator))) {
} else if (OB_FAIL(info.arg_->keys_.push_back(key))) {
LOG_WARN("fail to push back key", K(ret), K(key));
} else if (OB_ISNULL(buf = allocator.alloc(sizeof(ObArray<int64_t>)))) {
ret = OB_ALLOCATE_MEMORY_FAILED;
LOG_WARN("fail to alloc mem", K(ret));
} else if (FALSE_IT(info.idx_array_ = new (buf)
ObArray<int64_t>(OB_MALLOC_NORMAL_BLOCK_SIZE, ModulePageAllocator(allocator)))) {
} else if (OB_FAIL(info.idx_array_->push_back(i))) {
LOG_WARN("fail to push back idx", K(ret), K(key), K(i));
} else {
info.addr_ = addr;
if (OB_FAIL(infos.push_back(info))) {
LOG_WARN("fail to push back idx", K(ret), K(key), K(i));
} else if (OB_FAIL(dest_map.set_refactored(addr, infos.count() - 1))) {
LOG_WARN("fail to set value", K(ret), K(addr));
} else {
renew_by_sql = false;
}
}
}
} else if (idx >= infos.count() || idx < 0) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("invalid idx", K(ret), K(idx), "cnt", infos.count());
} else {
ObLocationRpcRenewInfo& tmp_info = infos.at(idx);
if (!tmp_info.is_valid()) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("invalid tmp_info", K(ret), K(tmp_info));
} else if (OB_FAIL(tmp_info.arg_->keys_.push_back(key))) {
LOG_WARN("fail to push back key", K(ret), K(key));
} else if (OB_FAIL(tmp_info.idx_array_->push_back(i))) {
LOG_WARN("fail to push back idx", K(ret), K(key), K(i));
} else {
renew_by_sql = false;
}
}
}
if (OB_SUCC(ret)) {
// need renew by sql, marked failed
int tmp_ret = renew_by_sql ? OB_SKIP_RENEW_LOCATION_BY_RPC : OB_SUCCESS;
if (OB_FAIL(key_results.push_back(tmp_ret))) { // init
LOG_WARN("fail to init key results", K(ret), K(i));
} else if (renew_by_sql) {
LOG_DEBUG("skip renew location by rpc", K(tmp_ret), K(table_id), K(partition_id));
}
}
}
}
if (OB_SUCC(ret) && key_results.count() != rpc_locations.count()) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("cnt not match", K(ret), "key_cnt", key_results.count(), "location_cnt", rpc_locations.count());
}
}
return ret;
}
ObPartitionLocationCache::LocationSem::LocationSem() : cur_count_(0), max_count_(0), cond_()
{
cond_.init(ObWaitEventIds::LOCATION_CACHE_COND_WAIT);
}
ObPartitionLocationCache::LocationSem::~LocationSem()
{}
void ObPartitionLocationCache::LocationSem::set_max_count(const int64_t max_count)
{
cond_.lock();
max_count_ = max_count;
cond_.unlock();
LOG_INFO("location cache fetch location concurrent max count changed", K(max_count));
}
int ObPartitionLocationCache::LocationSem::acquire(const int64_t abs_timeout_us)
{
// when we change max_count to small value, cur_count > max_count is possible
int ret = OB_SUCCESS;
const int64_t default_wait_time_ms = 1000;
int64_t wait_time_ms = default_wait_time_ms;
bool has_wait = false;
cond_.lock();
if (max_count_ <= 0 || cur_count_ < 0) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("invalid max_count", K(ret), K_(max_count), K_(cur_count));
} else {
while (OB_SUCC(ret) && cur_count_ >= max_count_) {
if (abs_timeout_us > 0) {
wait_time_ms = (abs_timeout_us - ObTimeUtility::current_time()) / 1000 + 1; // 1ms at least
if (wait_time_ms <= 0) {
ret = OB_TIMEOUT;
}
} else {
wait_time_ms = default_wait_time_ms;
}
if (OB_SUCC(ret)) {
if (wait_time_ms > INT32_MAX) {
wait_time_ms = INT32_MAX;
const bool force_print = true;
BACKTRACE(ERROR,
force_print,
"Invalid wait time, use INT32_MAX. wait_time_ms=%ld abs_timeout_us=%ld",
wait_time_ms,
abs_timeout_us);
}
has_wait = true;
cond_.wait(static_cast<int32_t>(wait_time_ms));
}
}
if (has_wait) {
EVENT_INC(LOCATION_CACHE_WAIT);
}
if (OB_SUCC(ret)) {
++cur_count_;
}
}
cond_.unlock();
return ret;
}
int ObPartitionLocationCache::LocationSem::release()
{
int ret = OB_SUCCESS;
cond_.lock();
if (cur_count_ <= 0) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("invalid cur_count", K(ret), K_(cur_count));
} else {
--cur_count_;
}
cond_.signal();
cond_.unlock();
return ret;
}
bool ObPartitionLocationCache::RenewLimiter::try_update_last_renew_timestamp(
const ObPartitionKey& key, int64_t renew_timestamp)
{
bool update_succ = false;
int64_t idx = key.hash() % PARTITION_HASH_BUCKET_COUNT;
int64_t old_last_renew_timestamp = last_renew_timestamps_[idx];
if (old_last_renew_timestamp ==
ATOMIC_VCAS(&(last_renew_timestamps_[idx]), old_last_renew_timestamp, renew_timestamp)) {
update_succ = true;
}
return update_succ;
}
static const char* location_queue_type_str_array[ObLocationCacheQueueSet::LOC_QUEUE_MAX + 1] = {
"SYS_CORE", "SYS_RESTART_RELATED", "SYS", "USER_HA", "USER_TENANT_SPACE", "USER", "MAX"};
const char* ObLocationCacheQueueSet::get_str_by_queue_type(Type type)
{
const char* str = NULL;
if (ObLocationCacheQueueSet::LOC_QUEUE_SYS_CORE <= type && type < ObLocationCacheQueueSet::LOC_QUEUE_MAX) {
str = location_queue_type_str_array[type];
} else {
str = location_queue_type_str_array[ObLocationCacheQueueSet::LOC_QUEUE_MAX];
}
return str;
}
ObLocationCacheQueueSet::ObLocationCacheQueueSet()
: is_inited_(false),
all_root_update_queue_(),
rs_restart_queue_(),
sys_update_queue_(),
user_ha_update_queue_(),
tenant_space_update_queue_(),
user_update_queue_tg_id_(-1)
{}
ObLocationCacheQueueSet::~ObLocationCacheQueueSet()
{
all_root_update_queue_.destroy();
rs_restart_queue_.destroy();
sys_update_queue_.destroy();
user_ha_update_queue_.destroy();
tenant_space_update_queue_.destroy();
TG_DESTROY(user_update_queue_tg_id_);
}
int ObLocationCacheQueueSet::init(ObServerConfig& config)
{
int ret = OB_SUCCESS;
if (is_inited_) {
ret = OB_INIT_TWICE;
LOG_WARN("queue set has already inited", K(ret));
} else if (OB_FAIL(all_root_update_queue_.init(
ALL_ROOT_THREAD_CNT, "RootLocUpdTask", PLC_TASK_QUEUE_SIZE, PLC_TASK_MAP_SIZE))) {
LOG_WARN("all_root_update_queue init failed", K(ret));
} else if (OB_FAIL(rs_restart_queue_.init(SYS_THREAD_CNT, "RsLocUpdTask", PLC_TASK_QUEUE_SIZE, PLC_TASK_MAP_SIZE))) {
LOG_WARN("rs_restart_queue init failed", K(ret));
} else if (OB_FAIL(sys_update_queue_.init(SYS_THREAD_CNT, "SysLocUpdTask", PLC_TASK_QUEUE_SIZE, PLC_TASK_MAP_SIZE))) {
LOG_WARN("sys_update_queue init failed", K(ret));
} else if (OB_FAIL(user_ha_update_queue_.init(
SYS_THREAD_CNT, "UsrHaLocUpdTask", PLC_TASK_QUEUE_SIZE, PLC_TASK_MAP_SIZE))) {
LOG_WARN("user_ha_update_queue init failed", K(ret));
} else if (OB_FAIL(tenant_space_update_queue_.init(static_cast<int32_t>(config.location_refresh_thread_count),
"TntLocUpdTask",
PLC_TASK_QUEUE_SIZE,
PLC_TASK_MAP_SIZE))) {
LOG_WARN("tenant_space_update_queue init failed", K(ret));
} else if (OB_FAIL(TG_CREATE(lib::TGDefIDs::UsrLocUpdTask, user_update_queue_tg_id_))) {
LOG_WARN("create tg failed", K(ret));
} else if (OB_FAIL(TG_START(user_update_queue_tg_id_))) {
LOG_WARN("start failed", K(ret));
} else {
all_root_update_queue_.set_label(ObModIds::OB_LOCATION_CACHE_QUEUE);
rs_restart_queue_.set_label(ObModIds::OB_LOCATION_CACHE_QUEUE);
sys_update_queue_.set_label(ObModIds::OB_LOCATION_CACHE_QUEUE);
user_ha_update_queue_.set_label(ObModIds::OB_LOCATION_CACHE_QUEUE);
tenant_space_update_queue_.set_label(ObModIds::OB_LOCATION_CACHE_QUEUE);
is_inited_ = true;
}
return ret;
}
ObLocationCacheQueueSet::Type ObLocationCacheQueueSet::get_queue_type(
const uint64_t table_id, const int64_t partition_id)
{
Type type = LOC_QUEUE_MAX;
if (table_id == combine_id(OB_SYS_TENANT_ID, OB_ALL_CORE_TABLE_TID) ||
table_id == combine_id(OB_SYS_TENANT_ID, OB_ALL_ROOT_TABLE_TID) ||
table_id == combine_id(OB_SYS_TENANT_ID, OB_ALL_GTS_TID) ||
table_id == combine_id(OB_SYS_TENANT_ID, OB_ALL_TENANT_GTS_TID)) {
type = LOC_QUEUE_SYS_CORE;
} else if (is_sys_table(table_id)) {
// sys table
if (extract_pure_id(table_id) == OB_ALL_TENANT_META_TABLE_TID || extract_pure_id(table_id) == OB_ALL_DUMMY_TID) {
// high priority in tenant space
type = LOC_QUEUE_USER_HA;
} else if (OB_SYS_TENANT_ID == extract_tenant_id(table_id)) {
// in sys tenant
if (ObSysTableChecker::is_rs_restart_related_partition(table_id, partition_id)) {
type = LOC_QUEUE_SYS_RESTART_RELATED;
} else {
type = LOC_QUEUE_SYS;
}
} else {
type = LOC_QUEUE_USER_TENANT_SPACE;
}
} else {
type = LOC_QUEUE_USER;
}
return type;
}
int ObLocationCacheQueueSet::add_task(const ObLocationUpdateTask& task)
{
int ret = OB_SUCCESS;
if (!is_inited_) {
ret = OB_NOT_INIT;
LOG_WARN("not init", K(ret));
} else if (!task.is_valid()) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("invalid task", K(ret), K(task));
} else {
Type type = get_queue_type(task.get_table_id(), task.get_partition_id());
switch (type) {
case LOC_QUEUE_SYS_CORE: {
if (OB_FAIL(all_root_update_queue_.add_task(task))) {
if (OB_EAGAIN != ret) {
LOG_WARN("all_root_update_queue add_task failed", K(task), K(ret));
} else {
ret = OB_SUCCESS;
}
}
break;
}
case LOC_QUEUE_SYS_RESTART_RELATED: {
if (OB_FAIL(rs_restart_queue_.add_task(task))) {
if (OB_EAGAIN != ret) {
LOG_WARN("rs_restart_queue add_task failed", K(task), K(ret));
} else {
ret = OB_SUCCESS;
}
}
break;
}
case LOC_QUEUE_SYS: {
if (OB_FAIL(sys_update_queue_.add_task(task))) {
if (OB_EAGAIN != ret) {
LOG_WARN("sys_update_queue add_task failed", K(task), K(ret));
} else {
ret = OB_SUCCESS;
}
}
break;
}
case LOC_QUEUE_USER_HA: {
if (OB_FAIL(user_ha_update_queue_.add_task(task))) {
if (OB_EAGAIN != ret) {
LOG_WARN("user_ha_update_queue add_task failed", K(task), K(ret));
} else {
ret = OB_SUCCESS;
}
}
break;
}
case LOC_QUEUE_USER_TENANT_SPACE: {
if (OB_FAIL(tenant_space_update_queue_.add_task(task))) {
if (OB_EAGAIN != ret) {
LOG_WARN("tenant_space_update_queue add_task failed", K(task), K(ret));
} else {
ret = OB_SUCCESS;
}
}
break;
}
case LOC_QUEUE_USER: {
if (OB_FAIL(TG_PUSH_TASK(user_update_queue_tg_id_, task))) {
if (OB_EAGAIN != ret) {
LOG_WARN("user_update_queue_ add_task failed", K(task), K(ret));
} else {
ret = OB_SUCCESS;
}
}
break;
}
default: {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("invalid queue type", KR(ret), K(task), K(type));
}
}
}
return ret;
}
ObLocationAsyncUpdateTask::ObLocationAsyncUpdateTask()
: loc_cache_(NULL),
table_id_(INT64_MAX),
partition_id_(OB_INVALID_ID),
add_timestamp_(OB_INVALID_VERSION),
cluster_id_(OB_INVALID_ID),
type_(MODE_AUTO)
{}
ObLocationAsyncUpdateTask::ObLocationAsyncUpdateTask(ObIPartitionLocationCache& loc_cache, const uint64_t table_id,
const int64_t partition_id, const int64_t add_timestamp, const int64_t cluster_id, const Type type)
: loc_cache_(&loc_cache),
table_id_(table_id),
partition_id_(partition_id),
add_timestamp_(add_timestamp),
cluster_id_(cluster_id),
type_(type)
{}
bool ObLocationAsyncUpdateTask::need_process_alone() const
{
bool bret = false;
if (GET_MIN_CLUSTER_VERSION() < CLUSTER_VERSION_2260 || OB_ALL_CORE_TABLE_TID == extract_pure_id(table_id_) ||
OB_ALL_ROOT_TABLE_TID == extract_pure_id(table_id_) || is_virtual_table(table_id_)) {
bret = true;
}
return bret;
}
bool ObLocationAsyncUpdateTask::is_valid() const
{
return OB_NOT_NULL(loc_cache_) && OB_INVALID_ID != table_id_ && OB_INVALID_ID != partition_id_ &&
OB_INVALID_VERSION != add_timestamp_ && OB_INVALID_ID != cluster_id_;
}
void ObLocationAsyncUpdateTask::check_task_status() const
{
const int64_t now = ObTimeUtility::current_time();
const int64_t wait_cost = now - add_timestamp_;
if (wait_cost >= WAIT_PROCESS_WARN_TIME) {
LOG_WARN("location update task waits to be processed too long", "task", *this, K(now), K(wait_cost));
}
}
int64_t ObLocationAsyncUpdateTask::hash() const
{
int64_t hash_val = 0;
if (!is_valid()) {
LOG_WARN("invalid argument", "self", *this);
} else {
hash_val = murmurhash(&table_id_, sizeof(table_id_), hash_val);
hash_val = murmurhash(&partition_id_, sizeof(partition_id_), hash_val);
hash_val = murmurhash(&cluster_id_, sizeof(cluster_id_), hash_val);
hash_val = murmurhash(&type_, sizeof(type_), hash_val);
}
return hash_val;
}
bool ObLocationAsyncUpdateTask::operator==(const ObLocationAsyncUpdateTask& other) const
{
bool equal = false;
if (!is_valid() || !other.is_valid()) {
LOG_WARN("invalid argument", "self", *this, K(other));
} else if (&other == this) {
equal = true;
} else {
equal = (table_id_ == other.table_id_ && partition_id_ == other.partition_id_ && cluster_id_ == other.cluster_id_ &&
type_ == other.type_);
}
return equal;
}
bool ObLocationAsyncUpdateTask::operator<(const ObLocationAsyncUpdateTask& other) const
{
return table_id_ < other.table_id_ || partition_id_ < other.partition_id_ || cluster_id_ < other.cluster_id_ ||
type_ < other.type_;
;
}
bool ObLocationAsyncUpdateTask::compare_without_version(const ObLocationAsyncUpdateTask& other) const
{
bool equal = false;
if (!is_valid() || !other.is_valid()) {
LOG_ERROR("invalid argument", "self", *this, K(other));
} else if (&other == this) {
equal = true;
} else {
equal = (*this == other);
}
return equal;
}
uint64_t ObLocationAsyncUpdateTask::get_group_id() const
{
return combine_id(extract_tenant_id(table_id_), type_);
}
bool ObLocationAsyncUpdateTask::is_barrier() const
{
return false;
}
// discard remote location cache refresh task while switchover/failover
bool ObLocationAsyncUpdateTask::need_discard() const
{
bool bret = false;
if (GCTX.is_in_flashback_state() || GCTX.is_in_cleanup_state()) {
if (cluster_id_ != GCONF.cluster_id) {
bret = true;
} else {
// nothing todo
}
}
return bret;
}
ObLocationAsyncUpdateQueueSet::ObLocationAsyncUpdateQueueSet(ObIPartitionLocationCache* loc_cache)
: is_inited_(false),
loc_cache_(loc_cache),
all_root_update_queue_(),
rs_restart_queue_(),
sys_update_queue_(),
user_ha_update_queue_(),
tenant_space_update_queue_(),
user_update_queue_()
{}
ObLocationAsyncUpdateQueueSet::~ObLocationAsyncUpdateQueueSet()
{}
void ObLocationAsyncUpdateQueueSet::stop()
{
all_root_update_queue_.stop();
rs_restart_queue_.stop();
sys_update_queue_.stop();
user_ha_update_queue_.stop();
tenant_space_update_queue_.stop();
user_update_queue_.stop();
}
void ObLocationAsyncUpdateQueueSet::wait()
{
all_root_update_queue_.wait();
rs_restart_queue_.wait();
sys_update_queue_.wait();
user_ha_update_queue_.wait();
tenant_space_update_queue_.wait();
user_update_queue_.wait();
}
int ObLocationAsyncUpdateQueueSet::init(ObServerConfig& config)
{
int ret = OB_SUCCESS;
int64_t user_thread_cnt =
!lib::is_mini_mode() ? static_cast<int32_t>(config.location_refresh_thread_count) : MINI_MODE_UPDATE_THREAD_CNT;
if (is_inited_) {
ret = OB_INIT_TWICE;
LOG_WARN("queue set has already inited", K(ret));
} else if (OB_FAIL(all_root_update_queue_.init(
loc_cache_, MINI_MODE_UPDATE_THREAD_CNT, PLC_TASK_QUEUE_SIZE, "RootLocAsyncUp"))) {
LOG_WARN("all_root_update_queue init failed", K(ret));
} else if (OB_FAIL(rs_restart_queue_.init(
loc_cache_, MINI_MODE_UPDATE_THREAD_CNT, PLC_TASK_QUEUE_SIZE, "RsLocAsyncUp"))) {
LOG_WARN("rs_restart_queue init failed", K(ret));
} else if (OB_FAIL(sys_update_queue_.init(
loc_cache_, MINI_MODE_UPDATE_THREAD_CNT, PLC_TASK_QUEUE_SIZE, "SysLocAsyncUp"))) {
LOG_WARN("sys_update_queue init failed", K(ret));
} else if (OB_FAIL(user_ha_update_queue_.init(
loc_cache_, MINI_MODE_UPDATE_THREAD_CNT, PLC_TASK_QUEUE_SIZE, "UsrHaLocAsyncUp"))) {
LOG_WARN("user_ha_update_queue init failed", K(ret));
} else if (OB_FAIL(
tenant_space_update_queue_.init(loc_cache_, user_thread_cnt, PLC_TASK_QUEUE_SIZE, "TntLocAsyncUp"))) {
LOG_WARN("tenant_space_update_queue init failed", K(ret));
} else if (OB_FAIL(user_update_queue_.init(loc_cache_,
user_thread_cnt,
!lib::is_mini_mode() ? USER_TASK_QUEUE_SIZE : MINI_MODE_USER_TASK_QUEUE_SIZE,
"UserLocAsyncUp"))) {
LOG_WARN("user_update_queue init failed", K(ret));
} else {
is_inited_ = true;
}
return ret;
}
int ObLocationAsyncUpdateQueueSet::add_task(const ObLocationAsyncUpdateTask& task)
{
int ret = OB_SUCCESS;
if (!is_inited_) {
ret = OB_NOT_INIT;
LOG_WARN("not init", K(ret));
} else if (!task.is_valid()) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("invalid task", K(ret), K(task));
} else {
const uint64_t table_id = task.get_table_id();
const int64_t partition_id = task.get_partition_id();
if (table_id == combine_id(OB_SYS_TENANT_ID, OB_ALL_CORE_TABLE_TID) ||
table_id == combine_id(OB_SYS_TENANT_ID, OB_ALL_ROOT_TABLE_TID) ||
table_id == combine_id(OB_SYS_TENANT_ID, OB_ALL_GTS_TID) ||
table_id == combine_id(OB_SYS_TENANT_ID, OB_ALL_TENANT_GTS_TID)) {
// high priority
if (OB_FAIL(all_root_update_queue_.add(task))) {
if (OB_EAGAIN != ret) {
LOG_WARN("all_root_update_queue add_task failed", K(task), K(ret));
} else {
ret = OB_SUCCESS;
}
}
} else if (is_sys_table(table_id)) {
// sys table
if (extract_pure_id(table_id) == OB_ALL_TENANT_META_TABLE_TID || extract_pure_id(table_id) == OB_ALL_DUMMY_TID) {
// high priority in tenant space
if (OB_FAIL(user_ha_update_queue_.add(task))) {
if (OB_EAGAIN != ret) {
LOG_WARN("user_ha_update_queue add_task failed", K(task), K(ret));
} else {
ret = OB_SUCCESS;
}
}
} else if (OB_SYS_TENANT_ID == task.get_tenant_id()) {
// in sys tenant
if (ObSysTableChecker::is_rs_restart_related_partition(table_id, partition_id)) {
if (OB_FAIL(rs_restart_queue_.add(task))) {
if (OB_EAGAIN != ret) {
LOG_WARN("rs_restart_queue add_task failed", K(task), K(ret));
} else {
ret = OB_SUCCESS;
}
}
} else {
if (OB_FAIL(sys_update_queue_.add(task))) {
if (OB_EAGAIN != ret) {
LOG_WARN("sys_update_queue add_task failed", K(task), K(ret));
} else {
ret = OB_SUCCESS;
}
}
}
} else {
if (OB_FAIL(tenant_space_update_queue_.add(task))) {
if (OB_EAGAIN != ret) {
LOG_WARN("tenant_space_update_queue add_task failed", K(task), K(ret));
} else {
ret = OB_SUCCESS;
}
}
}
} else {
if (OB_FAIL(user_update_queue_.add(task))) {
if (OB_EAGAIN != ret) {
LOG_WARN("user_update_queue add_task failed", K(task), K(ret));
} else {
ret = OB_SUCCESS;
}
}
}
}
return ret;
}
ObPartitionLocationCache::ObPartitionLocationCache(ObILocationFetcher& location_fetcher)
: is_inited_(false),
is_stopped_(false),
location_fetcher_(location_fetcher),
schema_service_(NULL),
config_(NULL),
sys_cache_(),
sys_leader_cache_(),
user_cache_(),
server_tracer_(NULL),
sem_(),
renew_limiter_(),
locality_manager_(NULL),
cluster_id_(-1),
remote_server_provider_(NULL),
lock_(),
sql_renew_map_(),
sql_renew_pool_(),
leader_cache_(),
local_async_queue_set_(this),
remote_async_queue_set_(this)
{}
ObPartitionLocationCache::~ObPartitionLocationCache()
{}
int ObPartitionLocationCache::init(ObMultiVersionSchemaService& schema_service, ObServerConfig& config,
ObIAliveServerTracer& server_tracer, const char* cache_name, const int64_t priority,
ObILocalityManager* locality_manager, const int64_t cluster_id, ObRemoteServerProvider* remote_server_provider)
{
int ret = OB_SUCCESS;
if (is_inited_) {
ret = OB_INIT_TWICE;
LOG_WARN("partition location cache has already inited", K(ret));
} else if (NULL == cache_name || priority <= 0 || OB_ISNULL(locality_manager) || OB_ISNULL(remote_server_provider)) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("invalid argument", K(ret), K(cache_name), K(priority), K(remote_server_provider));
} else {
const int64_t TENANT_COUNT = 128;
if (OB_FAIL(sys_cache_.create(OB_SYS_LOCATION_CACHE_BUCKET_NUM, ObModIds::OB_RS_SYS_LOCATION_CACHE))) {
LOG_WARN("create system table location cache failed", K(ret), LITERAL_K(OB_SYS_LOCATION_CACHE_BUCKET_NUM));
} else if (OB_FAIL(
sys_leader_cache_.create(OB_SYS_LOCATION_CACHE_BUCKET_NUM, ObModIds::OB_RS_SYS_LOCATION_CACHE))) {
LOG_WARN("create system table leader cache failed", K(ret), LITERAL_K(OB_SYS_LOCATION_CACHE_BUCKET_NUM));
} else if (OB_FAIL(user_cache_.init(cache_name, priority))) {
LOG_WARN("user table location cache init failed", K(cache_name), K(priority), K(ret));
} else if (OB_FAIL(sql_renew_map_.create(TENANT_COUNT, ObModIds::OB_RS_SYS_LOCATION_CACHE))) {
LOG_WARN("fail to init sql renew info", KR(ret));
} else if (OB_FAIL(local_async_queue_set_.init(config))) {
LOG_WARN("fail to init local async queue set", K(ret));
} else if (OB_FAIL(remote_async_queue_set_.init(config))) {
LOG_WARN("fail to init remote async queue set", K(ret));
} else {
sem_.set_max_count(config.location_fetch_concurrency);
schema_service_ = &schema_service;
config_ = &config;
server_tracer_ = &server_tracer;
locality_manager_ = locality_manager;
cluster_id_ = cluster_id;
is_stopped_ = false;
remote_server_provider_ = remote_server_provider;
is_inited_ = true;
}
}
return ret;
}
void ObPartitionLocationCache::stop()
{
local_async_queue_set_.stop();
remote_async_queue_set_.stop();
}
void ObPartitionLocationCache::wait()
{
local_async_queue_set_.wait();
remote_async_queue_set_.wait();
}
int ObPartitionLocationCache::destroy()
{
int ret = OB_SUCCESS;
if (OB_FAIL(sys_cache_.destroy())) {
LOG_WARN("destroy sys cache failed", K(ret));
} else if (OB_FAIL(sys_leader_cache_.destroy())) {
LOG_WARN("destroy sys cache failed", K(ret));
} else if (OB_FAIL(sql_renew_map_.destroy())) {
LOG_WARN("fail to clear map", KR(ret));
} else {
is_stopped_ = true;
is_inited_ = false;
}
return ret;
}
int ObPartitionLocationCache::reload_config()
{
int ret = OB_SUCCESS;
if (!is_inited_) {
ret = OB_NOT_INIT;
LOG_WARN("not init", K(ret));
} else {
sem_.set_max_count(config_->location_fetch_concurrency);
}
return ret;
}
bool ObPartitionLocationCache::is_duty_time(const ObPartitionLocation& location)
{
bool bret = false;
if (!location.is_valid()) {
// nothing todo
} else {
int64_t curr_time = ObTimeUtility::current_time();
bret = (curr_time - location.get_sql_renew_time()) >= config_->force_refresh_location_cache_interval;
LOG_TRACE("is duty time", K(bret), K(location), K(curr_time));
if (bret) {
LOG_TRACE("is duty time", K(bret), K(location), K(curr_time), K(curr_time - location.get_sql_renew_time()));
}
}
return bret;
}
bool ObPartitionLocationCache::can_do_force_sql_renew(const uint64_t table_id)
{
bool bret = false;
int ret = OB_SUCCESS;
int64_t tenant_id = extract_tenant_id(table_id);
ObTenantSqlRenewStat* tenant_info = NULL;
if (OB_INVALID_ID == table_id) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("invalid argument", KR(ret), K(table_id));
} else {
ObTenantStatGuard guard(tenant_id, sql_renew_map_, sql_renew_pool_, lock_);
tenant_info = guard.get_tenant_stat();
if (OB_ISNULL(tenant_info)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("get invalid tenant_info", KR(ret));
} else if (ObTimeUtility::current_time() - tenant_info->start_sql_renew_ts_ > FORCE_SQL_RENEW_WINDOW) {
bret = true;
tenant_info->init(tenant_id);
tenant_info->inc();
} else if (tenant_info->sql_renew_count_ < config_->force_refresh_location_cache_threshold) {
bret = true;
tenant_info->inc();
}
if (bret) {
LOG_TRACE("can do force sql renew", K(*tenant_info), K(bret));
}
}
return bret;
}
// interface for obproxy to renew location cache by SQL directly in some cases.
int ObPartitionLocationCache::force_sql_get(const uint64_t table_id, const int64_t partition_id,
ObPartitionLocation& location, const int64_t expire_renew_time, bool& is_cache_hit)
{
int ret = OB_SUCCESS;
is_cache_hit = false;
if (!is_inited_) {
ret = OB_NOT_INIT;
LOG_WARN("location cache is not inited", K(ret));
} else if (!ObIPartitionTable::is_valid_key(table_id, partition_id) || is_virtual_table(table_id)) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("table id or partition id is invalid", KT(table_id), K(partition_id), K(ret));
} else {
int64_t cluster_id = cluster_id_;
const bool filter_not_readable_replica = true;
const bool force_sql_renew = true;
const bool auto_update = true;
if (!can_do_force_sql_renew(table_id)) {
if (OB_FAIL(get(table_id, partition_id, location, expire_renew_time, is_cache_hit, auto_update))) {
LOG_WARN("fail to get", KR(ret), K(table_id), K(partition_id));
}
} else if (OB_FAIL(renew_location(table_id,
partition_id,
cluster_id,
location,
expire_renew_time,
filter_not_readable_replica,
force_sql_renew,
auto_update))) {
LOG_WARN("renew location failed",
K(ret),
KT(table_id),
K(partition_id),
K(expire_renew_time),
K(filter_not_readable_replica),
K(lbt()));
} else {
#if !defined(NDEBUG)
LOG_INFO("sync renew location by sql success",
K(ret),
KT(table_id),
K(partition_id),
K(location),
K(expire_renew_time),
K(filter_not_readable_replica),
K(lbt()));
#endif
}
}
// TODO: should monitor frequency of force_sql_refresh()
#if !defined(NDEBUG)
if (OB_FAIL(ret)) {
LOG_WARN("LOCATION: force get partition location", K(table_id), K(partition_id), K(ret));
} else {
LOG_INFO("LOCATION: force get partition location", K(table_id), K(partition_id), K(ret));
}
#endif
if (OB_ENTRY_NOT_EXIST == ret) {
ret = OB_LOCATION_NOT_EXIST;
}
if (OB_TIMEOUT == ret) {
ret = OB_GET_LOCATION_TIME_OUT;
}
return ret;
}
int ObPartitionLocationCache::get(const uint64_t table_id, const int64_t partition_id, ObPartitionLocation& location,
const int64_t expire_renew_time, bool& is_cache_hit, const bool auto_update /*= true*/)
{
int ret = OB_SUCCESS;
bool force_renew = true;
const bool is_nonblock = false;
is_cache_hit = false;
const bool filter_not_readable_replica = true;
int64_t expire_renew_time_new = expire_renew_time;
int64_t cluster_id = cluster_id_;
if (!is_inited_) {
ret = OB_NOT_INIT;
LOG_WARN("location cache is not inited", K(ret));
} else if (!ObIPartitionTable::is_valid_key(table_id, partition_id) || is_virtual_table(table_id)) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("table id or partition id is invalid", KT(table_id), K(partition_id), K(ret));
} else {
location.reset();
ret = get_from_cache(
table_id, partition_id, cluster_id, is_nonblock, location, filter_not_readable_replica, auto_update);
if (OB_SUCCESS != ret && OB_ENTRY_NOT_EXIST != ret) {
LOG_WARN("get location from cache failed", KT(table_id), K(partition_id), K(ret));
} else if (OB_ENTRY_NOT_EXIST == ret || location.is_mark_fail() || location.get_renew_time() <= expire_renew_time) {
force_renew = true;
is_cache_hit = false;
ret = OB_SUCCESS;
} else {
force_renew = false;
is_cache_hit = true;
bool alive = false;
int64_t trace_time = 0;
ObReplicaLocation leader;
if (OB_FAIL(location.get_strong_leader(leader))) {
if (OB_LOCATION_LEADER_NOT_EXIST == ret) {
ret = OB_SUCCESS;
} else {
LOG_WARN("get location leader failed", K(ret), K(location));
}
} else if (!is_reliable(location.get_renew_time()) && share::ObServerBlacklist::get_instance().is_in_blacklist(
share::ObCascadMember(leader.server_, cluster_id_))) {
LOG_DEBUG("leader in server blacklist, force renew", K(location));
force_renew = true;
expire_renew_time_new = location.get_renew_time();
} else if (OB_FAIL(server_tracer_->is_alive(leader.server_, alive, trace_time))) {
LOG_WARN("check server alive failed", K(ret), KT(table_id), K(partition_id), K(leader));
} else if (!alive) {
LOG_DEBUG("leader at non-alive server, force renew", K(location), K(trace_time));
force_renew = true;
expire_renew_time_new = trace_time;
}
}
// Determine whether to use a synchronous or asynchronous method to renew location by SQL
// when we force renew location by SQL.
bool force_sql_renew = false;
if (OB_SUCC(ret)) {
force_sql_renew = is_duty_time(location) && can_do_force_sql_renew(table_id);
const int64_t now = ObTimeUtility::current_time();
if (!force_sql_renew) {
// nothing todo
} else if (!force_renew) {
// cache is avaliable, renew location by SQL asynchronously.
ObLocationAsyncUpdateTask task(
*this, table_id, partition_id, now, cluster_id, ObLocationAsyncUpdateTask::MODE_SQL_ONLY);
// use temp_ret to avoid overwrite ret
int temp_ret = add_update_task(task);
if (OB_SUCCESS != temp_ret) {
LOG_WARN("add location update task failed", K(location), K(task), K(temp_ret));
} else {
LOG_INFO("add_update_task success", K(task));
}
} else {
// cache is avaliable, renew location by SQL synchronously.
}
}
if (OB_SUCC(ret) && force_renew) {
is_cache_hit = false;
int64_t cluster_id = cluster_id_;
if (OB_FAIL(renew_location(table_id,
partition_id,
cluster_id,
location,
expire_renew_time_new,
filter_not_readable_replica,
force_sql_renew,
auto_update))) {
LOG_WARN("renew location failed",
K(ret),
KT(table_id),
K(partition_id),
K(expire_renew_time_new),
K(filter_not_readable_replica),
K(lbt()));
} else {
#if !defined(NDEBUG)
LOG_INFO("sync renew location success",
K(ret),
KT(table_id),
K(partition_id),
K(location),
K(expire_renew_time),
K(filter_not_readable_replica),
K(force_renew),
K(lbt()));
#endif
}
}
}
if (OB_SUCC(ret) && is_cache_hit) {
EVENT_INC(LOCATION_CACHE_HIT);
} else {
EVENT_INC(LOCATION_CACHE_MISS);
}
#if !defined(NDEBUG)
if (OB_FAIL(ret)) {
LOG_WARN("LOCATION: get partition location",
K(table_id),
K(partition_id),
K(expire_renew_time),
K(force_renew),
K(is_cache_hit),
K(location),
K(ret));
} else {
LOG_INFO("LOCATION: get partition location",
K(table_id),
K(partition_id),
K(expire_renew_time),
K(force_renew),
K(is_cache_hit),
K(location),
K(ret));
}
#endif
if (OB_ENTRY_NOT_EXIST == ret) {
ret = OB_LOCATION_NOT_EXIST;
}
if (OB_TIMEOUT == ret) {
ret = OB_GET_LOCATION_TIME_OUT;
}
return ret;
}
// FIXME: (): to be removed
int ObPartitionLocationCache::get_across_cluster(const ObPartitionKey& partition, const int64_t expire_renew_time,
ObPartitionLocation& location, const int64_t specific_cluster_id)
{
int ret = OB_SUCCESS;
int64_t table_id = partition.get_table_id();
int64_t partition_id = partition.get_partition_id();
location.reset();
int64_t cluster_id = OB_INVALID_ID;
if (!is_inited_) {
ret = OB_NOT_INIT;
LOG_WARN("location cache is not inited", K(ret));
} else if (is_virtual_table(table_id)) {
ret = OB_OP_NOT_ALLOW;
LOG_WARN("virtual table should no use this function", KR(ret), K(partition));
} else {
cluster_id = OB_INVALID_ID == specific_cluster_id ? cluster_id_ : specific_cluster_id;
}
const bool filter_not_readable_replica = true;
int64_t expire_renew_time_new = expire_renew_time;
const bool is_nonblock = false;
bool force_renew = true;
bool auto_update = true;
if (OB_FAIL(ret)) {
} else {
ret = get_from_cache(
table_id, partition_id, cluster_id, is_nonblock, location, filter_not_readable_replica, auto_update);
if (OB_SUCCESS != ret && OB_ENTRY_NOT_EXIST != ret) {
LOG_WARN("get location from cache failed", K(partition), K(ret));
} else if (OB_ENTRY_NOT_EXIST == ret || location.is_mark_fail() || location.get_renew_time() <= expire_renew_time) {
force_renew = true;
ret = OB_SUCCESS;
} else {
force_renew = false;
}
if (OB_SUCC(ret) && force_renew) {
const bool force_sql_renew = false;
if (OB_FAIL(renew_location(table_id,
partition_id,
cluster_id,
location,
expire_renew_time_new,
filter_not_readable_replica,
force_sql_renew,
auto_update))) {
LOG_WARN("renew location failed",
K(ret),
KT(table_id),
K(partition_id),
K(expire_renew_time_new),
K(filter_not_readable_replica),
K(lbt()));
} else {
#if !defined(NDEBUG)
LOG_INFO("sync renew location success",
K(ret),
KT(table_id),
K(partition_id),
K(location),
K(expire_renew_time),
K(filter_not_readable_replica),
K(lbt()));
#endif
}
}
}
#if !defined(NDEBUG)
if (OB_FAIL(ret)) {
LOG_WARN("LOCATION: get partition location across",
K(table_id),
K(partition_id),
K(expire_renew_time),
K(force_renew),
K(ret));
} else {
LOG_INFO("LOCATION: get partition location across",
K(table_id),
K(partition_id),
K(expire_renew_time),
K(force_renew),
K(ret));
}
#endif
if (OB_ENTRY_NOT_EXIST == ret) {
ret = OB_LOCATION_NOT_EXIST;
}
if (OB_TIMEOUT == ret) {
ret = OB_GET_LOCATION_TIME_OUT;
}
return ret;
}
int ObPartitionLocationCache::get(
const ObPartitionKey& partition, ObPartitionLocation& location, const int64_t expire_renew_time, bool& is_cache_hit)
{
int ret = OB_SUCCESS;
if (!is_inited_) {
ret = OB_NOT_INIT;
LOG_WARN("not init", K(ret));
} else if (!partition.is_valid()) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("invalid argument", K(ret), K(partition));
} else if (OB_FAIL(get(
partition.get_table_id(), partition.get_partition_id(), location, expire_renew_time, is_cache_hit))) {
LOG_WARN("get partition location failed", K(ret), K(partition), K(expire_renew_time));
}
return ret;
}
int ObPartitionLocationCache::get(const uint64_t table_id, ObIArray<ObPartitionLocation>& locations,
const int64_t expire_renew_time, bool& is_cache_hit, const bool auto_update /*=true*/)
{
int ret = OB_SUCCESS;
if (!is_inited_) {
ret = OB_NOT_INIT;
LOG_WARN("location cache is not inited", K(ret));
} else {
if (!is_virtual_table(table_id)) {
const ObTableSchema* table = NULL;
ObSchemaGetterGuard schema_guard;
const uint64_t tenant_id = extract_tenant_id(table_id);
if (OB_FAIL(schema_service_->get_tenant_schema_guard(tenant_id, schema_guard))) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("get schema guard failed", K(ret));
} else if (OB_FAIL(schema_guard.get_table_schema(table_id, table))) {
LOG_WARN("get table schema failed", K(table_id), K(ret));
} else if (OB_ISNULL(table)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("NULL ptr", K(table_id), K(ret));
} else {
locations.reset();
ObPartitionLocation location;
bool check_dropped_schema = false; // For SQL only, we won't get delay-deleted partitions.
schema::ObTablePartitionKeyIter pkey_iter(*table, check_dropped_schema);
int64_t partition_id = 0;
while (OB_SUCC(ret)) {
location.reset();
if (OB_FAIL(pkey_iter.next_partition_id_v2(partition_id))) {
if (ret != OB_ITER_END) {
LOG_WARN("Failed to get next partition id", K(table_id), K(ret));
}
} else if (OB_FAIL(get(table_id, partition_id, location, expire_renew_time, is_cache_hit, auto_update))) {
LOG_WARN("get location failed", KT(table_id), K(partition_id), K(ret));
} else if (OB_FAIL(locations.push_back(location))) {
LOG_WARN("push back location failed", K(ret));
}
}
if (OB_ITER_END == ret) {
ret = OB_SUCCESS;
}
}
} else {
locations.reset();
if (OB_FAIL(vtable_get(table_id, locations, expire_renew_time, is_cache_hit))) {
LOG_WARN("vtable_get failed", KT(table_id), K(expire_renew_time), K(ret));
}
}
}
if (OB_ENTRY_NOT_EXIST == ret) {
ret = OB_LOCATION_NOT_EXIST;
}
if (OB_TIMEOUT == ret) {
ret = OB_GET_LOCATION_TIME_OUT;
}
return ret;
}
// FIXME: () For storage only, to be removed
int ObPartitionLocationCache::get_leader_across_cluster(
const common::ObPartitionKey& partition, common::ObAddr& leader, const int64_t cluster_id, const bool force_renew)
{
int ret = OB_SUCCESS;
ObPartitionLocation location;
ObReplicaLocation replica_location;
int64_t expire_renew_time = force_renew ? INT64_MAX : 0; // INT64_MAX will force renew location
if (!is_inited_) {
ret = OB_NOT_INIT;
LOG_WARN("not init", K(ret));
} else if (!partition.is_valid()) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("invalid argument", K(ret), K(partition));
} else if (OB_FAIL(get_across_cluster(partition, expire_renew_time, location, cluster_id))) {
LOG_WARN("get failed", K(partition), K(force_renew), K(expire_renew_time), K(ret));
} else if (OB_FAIL(location.get_leader_by_election(replica_location))) {
LOG_WARN("location.get_leader failed", K(location), K(ret));
} else {
leader = replica_location.server_;
}
return ret;
}
int ObPartitionLocationCache::get_strong_leader(
const common::ObPartitionKey& partition, common::ObAddr& leader, const bool force_renew)
{
int ret = OB_SUCCESS;
ObPartitionLocation location;
ObReplicaLocation replica_location;
int64_t expire_renew_time = force_renew ? INT64_MAX : 0; // INT64_MAX will force renew location
bool is_cache_hit = false;
if (!is_inited_) {
ret = OB_NOT_INIT;
LOG_WARN("not init", K(ret));
} else if (!partition.is_valid()) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("invalid argument", K(ret), K(partition));
} else if (OB_FAIL(get(partition, location, expire_renew_time, is_cache_hit))) {
LOG_WARN("get failed", K(partition), K(force_renew), K(expire_renew_time), K(ret));
}
if (OB_FAIL(ret)) {
// nothing todo
} else if (OB_FAIL(location.get_strong_leader(replica_location))) {
LOG_WARN("location.get_leader failed", K(location), K(ret));
} else if (is_reliable(location.get_renew_time())) {
leader = replica_location.server_;
} else if (!share::ObServerBlacklist::get_instance().is_in_blacklist(
share::ObCascadMember(replica_location.server_, cluster_id_))) {
leader = replica_location.server_;
} else {
ret = OB_LOCATION_LEADER_NOT_EXIST;
LOG_DEBUG("old partition leader in server blacklist", KR(ret), K(partition), K(replica_location));
}
return ret;
}
int ObPartitionLocationCache::get_leader_by_election(
const common::ObPartitionKey& partition, common::ObAddr& leader, const bool force_renew)
{
int ret = OB_SUCCESS;
ObPartitionLocation location;
ObReplicaLocation replica_location;
int64_t expire_renew_time = force_renew ? INT64_MAX : 0; // INT64_MAX will force renew location
bool is_cache_hit = false;
if (!is_inited_) {
ret = OB_NOT_INIT;
LOG_WARN("not init", K(ret));
} else if (!partition.is_valid()) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("invalid argument", K(ret), K(partition));
} else if (OB_FAIL(get(partition, location, expire_renew_time, is_cache_hit))) {
LOG_WARN("get failed", K(partition), K(force_renew), K(expire_renew_time), K(ret));
}
if (OB_FAIL(ret)) {
// nothing todo
} else if (OB_FAIL(location.get_leader_by_election(replica_location))) {
LOG_WARN("location get standby leader failed", K(location), K(ret));
} else {
leader = replica_location.server_;
}
return ret;
}
int ObPartitionLocationCache::nonblock_get(
const uint64_t table_id, const int64_t partition_id, ObPartitionLocation& location, const int64_t cluster_id)
{
int ret = OB_SUCCESS;
const bool is_nonblock = true;
const bool filter_not_readable_replica = true;
if (!is_inited_) {
ret = OB_NOT_INIT;
LOG_WARN("location cache is not inited", K(ret));
} else if (!ObIPartitionTable::is_valid_key(table_id, partition_id) || is_virtual_table(table_id)) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("table id or partition id is invalid", KT(table_id), K(partition_id), K(ret));
} else {
location.reset();
int64_t tmp_cluster_id = (cluster_id == -1) ? cluster_id_ : cluster_id;
bool auto_update = true;
ret = get_from_cache(
table_id, partition_id, tmp_cluster_id, is_nonblock, location, filter_not_readable_replica, auto_update);
if (OB_SUCCESS != ret && OB_ENTRY_NOT_EXIST != ret) {
LOG_WARN("get location from cache failed", KT(table_id), K(partition_id), K(ret));
}
if (OB_SUCC(ret)) {
ObReplicaLocation replica_location;
if (OB_SUCCESS != location.get_leader_by_election(replica_location)) {
// ignore error log and ret
} else if (!is_reliable(location.get_renew_time()) &&
share::ObServerBlacklist::get_instance().is_in_blacklist(
share::ObCascadMember(replica_location.server_, cluster_id_))) {
ret = OB_LOCATION_NOT_EXIST;
LOG_DEBUG(
"old partition leader in server blacklist", KR(ret), K(table_id), K(partition_id), K(replica_location));
}
}
}
if (OB_SUCC(ret)) {
EVENT_INC(LOCATION_CACHE_NONBLOCK_HIT);
}
if (OB_ENTRY_NOT_EXIST == ret) {
ret = OB_LOCATION_NOT_EXIST;
EVENT_INC(LOCATION_CACHE_NONBLOCK_MISS);
}
return ret;
}
int ObPartitionLocationCache::nonblock_get(
const ObPartitionKey& partition, ObPartitionLocation& location, const int64_t cluster_id)
{
int ret = OB_SUCCESS;
if (!is_inited_) {
ret = OB_NOT_INIT;
LOG_WARN("not init", K(ret));
} else if (!partition.is_valid()) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("invalid argument", K(ret), K(partition));
} else if (OB_FAIL(nonblock_get(partition.get_table_id(), partition.get_partition_id(), location, cluster_id))) {
if (OB_LOCATION_NOT_EXIST != ret && OB_LOCATION_LEADER_NOT_EXIST != ret) {
LOG_WARN("non-block get failed", K(ret), K(partition));
}
}
return ret;
}
int ObPartitionLocationCache::nonblock_get_strong_leader(const ObPartitionKey& partition, ObAddr& leader)
{
int ret = OB_SUCCESS;
ObPartitionLocation location;
ObReplicaLocation replica_location;
int64_t cluster_id = cluster_id_;
LocationInfo leader_location_info;
if (!is_inited_) {
ret = OB_NOT_INIT;
LOG_WARN("not init", K(ret));
} else if (!partition.is_valid()) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("invalid argument", K(ret), K(partition));
} else if (use_sys_leader_cache(partition.get_table_id(), cluster_id_)) {
ObLocationCacheKey key(partition.get_table_id(), partition.get_partition_id(), cluster_id_);
int hash_ret = sys_leader_cache_.get_refactored(key, leader_location_info);
if (OB_SUCCESS == hash_ret) {
ret = leader_location_info.is_valid() ? hash_ret : OB_LOCATION_LEADER_NOT_EXIST;
} else if (OB_HASH_NOT_EXIST == hash_ret) {
ret = OB_LOCATION_LEADER_NOT_EXIST;
} else {
ret = hash_ret;
LOG_WARN("nonblock get leader failed", K(ret), K(key));
}
} else {
bool use_cache = !is_sys_table(partition.get_table_id());
// try get leader from cache
if (use_cache) {
ObLocationCacheKey key(partition.get_table_id(), partition.get_partition_id(), cluster_id_);
ret = leader_cache_.get_strong_leader_info(key, leader_location_info);
}
if (OB_FAIL(ret) || !use_cache) {
// overwrite ret
if (OB_FAIL(nonblock_get(partition, location, cluster_id))) {
LOG_WARN("nonblock get", K(ret), K(partition), K(location), K(cluster_id));
if (OB_LOCATION_LEADER_NOT_EXIST != ret && OB_LOCATION_NOT_EXIST != ret) {
LOG_WARN("get failed", K(partition), K(ret));
}
} else if (OB_FAIL(location.get_strong_leader(replica_location))) {
if (OB_LOCATION_LEADER_NOT_EXIST != ret) {
LOG_WARN("location.get_leader failed", K(location), K(ret));
}
} else {
leader_location_info.server_ = replica_location.server_;
leader_location_info.renew_ts_ = location.get_renew_time();
if (use_cache) {
// try update cache
int tmp_ret = OB_SUCCESS;
bool force_update = true;
ObLocationCacheKey key(partition.get_table_id(), partition.get_partition_id(), cluster_id_);
if (OB_SUCCESS != (tmp_ret = leader_cache_.set_strong_leader_info(key, leader_location_info, force_update))) {
LOG_WARN("fail to set leader info", K(tmp_ret), K(key), K(leader_location_info));
}
}
}
}
}
if (OB_SUCC(ret)) {
if (is_reliable(leader_location_info.renew_ts_)) {
// renew location cache just now
leader = leader_location_info.server_;
} else if (!share::ObServerBlacklist::get_instance().is_in_blacklist(
share::ObCascadMember(leader_location_info.server_, cluster_id_))) {
// not in server blacklist
leader = leader_location_info.server_;
} else {
// not a new location info and in server blacklist
ret = OB_LOCATION_LEADER_NOT_EXIST;
LOG_DEBUG(
"old partition leader in server blacklist", KR(ret), K(partition), "replica_location", leader_location_info);
}
}
return ret;
}
int ObPartitionLocationCache::nonblock_get_restore_leader(const ObPartitionKey& partition, ObAddr& leader)
{
int ret = OB_SUCCESS;
ObPartitionLocation location;
ObReplicaLocation replica_location;
int64_t cluster_id = cluster_id_;
if (!is_inited_) {
ret = OB_NOT_INIT;
LOG_WARN("not init", K(ret));
} else if (!partition.is_valid()) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("invalid argument", K(ret), K(partition));
} else if (OB_FAIL(nonblock_get(partition, location, cluster_id))) {
LOG_WARN("nonblock get", K(ret), K(partition), K(location), K(cluster_id));
if (OB_LOCATION_LEADER_NOT_EXIST != ret && OB_LOCATION_NOT_EXIST != ret) {
LOG_WARN("get failed", K(partition), K(ret));
}
} else if (OB_FAIL(location.get_restore_leader(replica_location))) {
if (OB_LOCATION_LEADER_NOT_EXIST != ret) {
LOG_WARN("location.get_leader failed", K(location), K(ret));
}
} else {
leader = replica_location.server_;
}
return ret;
}
int ObPartitionLocationCache::nonblock_get_leader_by_election(const ObPartitionKey& partition, ObAddr& leader)
{
int ret = OB_SUCCESS;
ObPartitionLocation location;
ObReplicaLocation replica_location;
int64_t cluster_id = cluster_id_;
if (!is_inited_) {
ret = OB_NOT_INIT;
LOG_WARN("not init", K(ret));
} else if (!partition.is_valid()) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("invalid argument", K(ret), K(partition));
} else if (OB_FAIL(nonblock_get(partition, location, cluster_id))) {
if (OB_LOCATION_LEADER_NOT_EXIST != ret && OB_LOCATION_NOT_EXIST != ret) {
LOG_WARN("get failed", K(partition), K(ret));
}
} else if (OB_FAIL(location.get_leader_by_election(replica_location))) {
if (OB_LOCATION_LEADER_NOT_EXIST != ret) {
LOG_WARN("location.get_leader failed", K(location), K(ret));
}
} else {
leader = replica_location.server_;
}
return ret;
}
int ObPartitionLocationCache::nonblock_get_strong_leader_without_renew(
const common::ObPartitionKey& partition, common::ObAddr& leader, const int64_t specific_cluster_id)
{
int ret = OB_SUCCESS;
ObPartitionLocation location;
ObReplicaLocation replica_location;
int64_t cluster_id = OB_INVALID_ID == specific_cluster_id ? cluster_id_ : specific_cluster_id;
if (!is_inited_) {
ret = OB_NOT_INIT;
LOG_WARN("not init", K(ret));
} else if (!partition.is_valid()) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("invalid argument", K(ret), K(partition));
} else if (OB_FAIL(
inner_get_from_cache(partition.get_table_id(), partition.get_partition_id(), cluster_id, location))) {
if (OB_ENTRY_NOT_EXIST != ret) {
LOG_WARN("get failed", K(partition), K(ret));
}
} else if (OB_FAIL(location.get_strong_leader(replica_location))) {
if (OB_LOCATION_LEADER_NOT_EXIST != ret) {
LOG_WARN("location.get_leader failed", K(location), K(ret));
}
} else {
leader = replica_location.server_;
}
return ret;
}
int ObPartitionLocationCache::nonblock_get_leader_by_election_without_renew(
const common::ObPartitionKey& partition, common::ObAddr& leader, const int64_t specific_cluster_id)
{
int ret = OB_SUCCESS;
ObPartitionLocation location;
ObReplicaLocation replica_location;
int64_t cluster_id = OB_INVALID_ID == specific_cluster_id ? cluster_id_ : specific_cluster_id;
if (!is_inited_) {
ret = OB_NOT_INIT;
LOG_WARN("not init", K(ret));
} else if (!partition.is_valid()) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("invalid argument", K(ret), K(partition));
} else if (OB_FAIL(
inner_get_from_cache(partition.get_table_id(), partition.get_partition_id(), cluster_id, location))) {
if (OB_ENTRY_NOT_EXIST != ret) {
LOG_WARN("get failed", K(partition), K(ret));
}
} else if (OB_FAIL(location.get_leader_by_election(replica_location))) {
if (OB_LOCATION_LEADER_NOT_EXIST != ret) {
LOG_WARN("location.get_leader failed", K(location), K(ret));
}
} else {
leader = replica_location.server_;
}
return ret;
}
int ObPartitionLocationCache::nonblock_renew(
const ObPartitionKey& partition, const int64_t expire_renew_time, const int64_t specific_cluster_id)
{
int ret = OB_SUCCESS;
if (!is_inited_) {
ret = OB_NOT_INIT;
LOG_WARN("not init", K(ret));
} else if (!partition.is_valid()) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("invalid partition key", K(partition), K(ret));
} else {
int64_t cluster_id = OB_INVALID_ID == specific_cluster_id ? cluster_id_ : specific_cluster_id;
const int64_t now = ObTimeUtility::current_time();
ObLocationAsyncUpdateTask task(*this,
partition.get_table_id(),
partition.get_partition_id(),
now,
cluster_id,
ObLocationAsyncUpdateTask::MODE_AUTO);
if (expire_renew_time > 0) {
if (!is_virtual_table(partition.get_table_id())) {
/*
* is_mark_failed_ is not used any more.
* if (OB_FAIL(clear_location(partition.get_table_id(), partition.get_partition_id(),
* expire_renew_time, cluster_id))) {
* LOG_WARN("clear_location failed", K(ret), K(partition), K(expire_renew_time));
* }
*/
} else {
if (OB_FAIL(clear_vtable_location(partition.get_table_id(), expire_renew_time))) {
LOG_WARN("clear_vtable_location failed", K(ret), "table_id", partition.get_table_id(), K(expire_renew_time));
}
}
if (OB_SUCC(ret)) {
LOG_TRACE("clear_location succeed", K(partition), K(expire_renew_time));
}
}
if (OB_SUCC(ret)) {
if (OB_FAIL(add_update_task(task))) {
LOG_WARN("add location update task failed", K(task), K(ret));
} else if (EXECUTE_COUNT_PER_SEC(16)) {
LOG_INFO("add_update_task succeed", K(partition), K(task));
}
}
}
return ret;
}
// if is_limited = true, it means nonblock_renew is not triggered.
int ObPartitionLocationCache::nonblock_renew_with_limiter(
const ObPartitionKey& partition, const int64_t expire_renew_time, bool& is_limited)
{
int ret = OB_SUCCESS;
is_limited = true;
int64_t cur_timestamp = ObTimeUtility::current_time();
;
int64_t last_renew_timestamp = renew_limiter_.get_last_renew_timestamp(partition);
if (cur_timestamp - last_renew_timestamp > RenewLimiter::RENEW_INTERVAL) {
if (renew_limiter_.try_update_last_renew_timestamp(partition, cur_timestamp)) {
is_limited = false;
if (OB_FAIL(nonblock_renew(partition, expire_renew_time))) {
LOG_WARN("fail to nonblock renew", K(ret), K(partition), K(expire_renew_time));
}
}
}
return ret;
}
int ObPartitionLocationCache::inner_get_from_cache(
const uint64_t table_id, const int64_t partition_id, const int64_t cluster_id, ObPartitionLocation& result)
{
int ret = OB_SUCCESS;
ObLocationCacheKey cache_key(table_id, partition_id, cluster_id);
result.reset();
if (!is_inited_) {
ret = OB_NOT_INIT;
LOG_WARN("not init", K(ret));
} else if (!ObIPartitionTable::is_valid_key(table_id, partition_id)) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("invalid argument", K(ret), KT(table_id), K(partition_id));
} else {
if (use_sys_cache(table_id)) {
int hash_ret = sys_cache_.get_refactored(cache_key, result);
if (OB_SUCCESS == hash_ret) {
LOG_TRACE("location hit in sys cache", K(cache_key), K(ret));
} else if (OB_HASH_NOT_EXIST == hash_ret) {
ret = OB_ENTRY_NOT_EXIST;
LOG_TRACE("location is not in sys cache", K(cache_key), K(ret));
} else {
ret = hash_ret;
LOG_WARN("get location from sys cache failed", K(cache_key), K(ret));
}
} else {
ObKVCacheHandle handle;
const ObLocationCacheValue* cache_value = NULL;
ret = user_cache_.get(cache_key, cache_value, handle);
if (OB_FAIL(ret)) {
if (OB_ENTRY_NOT_EXIST != ret) {
LOG_WARN("get location from user cache failed", K(cache_key), K(ret));
}
} else {
if (OB_FAIL(cache_value2location(*cache_value, result))) {
LOG_WARN("transform cache value to location failed", K(cache_value), K(ret));
}
}
}
}
return ret;
}
int ObPartitionLocationCache::get_from_cache(const uint64_t table_id, const int64_t partition_id,
const int64_t cluster_id, const bool is_nonblock, ObPartitionLocation& result,
const bool filter_not_readable_replica, const bool auto_update)
{
int ret = OB_SUCCESS;
// NG_TRACE(plc_get_from_cache_begin);
ObLocationCacheKey cache_key(table_id, partition_id, cluster_id);
ObPartitionLocation location;
if (!is_inited_) {
ret = OB_NOT_INIT;
LOG_WARN("not init", K(ret));
} else if (!ObIPartitionTable::is_valid_key(table_id, partition_id)) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("invalid argument", K(ret), KT(table_id), K(partition_id));
} else if (OB_FAIL(inner_get_from_cache(table_id, partition_id, cluster_id, location))) {
if (OB_ENTRY_NOT_EXIST != ret) {
LOG_WARN("fail to get from cache", KR(ret), K(table_id), K(partition_id), K(cluster_id));
}
}
// only nonblock get and leader not alive need add async update task here
bool leader_inactive_async_update = false;
bool leader_server_exist = true;
int64_t trace_time = 0;
bool alive = true;
ObReplicaLocation leader;
bool new_mode = GET_MIN_CLUSTER_VERSION() >= CLUSTER_VERSION_2274;
if (OB_FAIL(ret) || cluster_id != cluster_id_) {
} else if ((!new_mode && OB_FAIL(location.get_strong_leader(leader))) ||
(new_mode && OB_FAIL(location.get_leader_by_election(leader)))) {
// ignore result code
ret = OB_SUCCESS;
} else if (OB_FAIL(server_tracer_->get_server_status(leader.server_, alive, leader_server_exist, trace_time))) {
LOG_WARN("fail to get server status", KR(ret), K(leader));
}
if (!GCTX.is_standby_cluster() || new_mode || ObMultiClusterUtil::is_cluster_private_table(table_id)) {
if (OB_SUCC(ret) && !location.is_mark_fail() && is_nonblock && cluster_id_ == cluster_id) {
// check leader alive
if ((!new_mode && OB_FAIL(location.get_strong_leader(leader))) ||
(new_mode && OB_FAIL(location.get_leader_by_election(leader)))) {
if (OB_LOCATION_LEADER_NOT_EXIST == ret) {
ret = OB_SUCCESS;
leader_inactive_async_update = true;
} else {
LOG_WARN("get location leader failed", K(ret), K(location));
}
} else if (!alive || !leader_server_exist ||
share::ObServerBlacklist::get_instance().is_in_blacklist(
share::ObCascadMember(leader.server_, cluster_id_))) {
leader_inactive_async_update = true;
}
}
}
const int64_t now = ObTimeUtility::current_time();
if (OB_SUCC(ret) && auto_update) {
bool has_task = false;
if (is_nonblock && (cluster_id_ == cluster_id) && is_duty_time(location) && can_do_force_sql_renew(table_id)) {
// To resolve problem of imprecise rpc renew result for non paxos members,
// try generate asynchronous sql renew task if it's been long time since last renew location by SQL.
ObLocationAsyncUpdateTask task(
*this, table_id, partition_id, now, cluster_id, ObLocationAsyncUpdateTask::MODE_SQL_ONLY);
// use temp_ret to avoid overwrite ret
int temp_ret = add_update_task(task);
if (OB_SUCCESS != temp_ret) {
LOG_WARN("add location update task failed", K(location), K(task), K(temp_ret));
} else {
has_task = true;
LOG_INFO("add_update_task success", K(task), K(location));
}
}
// if location cache expires, add a update task. ObDequeQueue will filter duplicate task
// if location is mark fail and is_nonblock is false, renew location will be called by caller, no need to add upadte
// task if location leader on non-alive server and is nonblock get, need add a update task
if (OB_SUCCESS == ret && !has_task && (!location.is_mark_fail() || is_nonblock || leader_inactive_async_update)) {
if (now - location.get_renew_time() >= config_->location_cache_expire_time || location.is_mark_fail() ||
leader_inactive_async_update) {
ObLocationAsyncUpdateTask task(
*this, table_id, partition_id, now, cluster_id, ObLocationAsyncUpdateTask::MODE_AUTO);
// use temp_ret to avoid overwrite ret
int temp_ret = add_update_task(task);
if (OB_SUCCESS != temp_ret) {
LOG_WARN("add location update task failed", K(location), K(task), K(temp_ret));
} else {
LOG_INFO(
"add_update_task success", K(task), K(leader_inactive_async_update), K(now), K(location), K(is_nonblock));
}
}
}
}
if (OB_SUCC(ret) && is_nonblock && location.is_mark_fail()) {
ret = OB_ENTRY_NOT_EXIST;
LOG_DEBUG("location is mark fail, treat it as not exist for nonblock get", K(ret), K(location));
}
// NG_TRACE(plc_get_from_cache_end);
// filter not readable replica
int64_t safe_interval = 3 * ObAliveServerRefreshTask::REFRESH_INTERVAL_US; // 15S
if (OB_FAIL(ret)) {
// nothing todo
} else if (leader_server_exist) {
} else if (now - trace_time < safe_interval) {
// AliveServerTracer is credible enough if AliveServerTracer has refreshed in the recent period.
// When server is inactive, using an inactive leader to query may cause -4012 error.
// To avoid this, we try to reset leader to follower in location cache,
// which may force SQL/Trans layer to renew location before they use old location cache.
leader.role_ = FOLLOWER;
int64_t tmp_ret = OB_SUCCESS;
if (OB_SUCCESS != (tmp_ret = location.update(leader.server_, leader))) {
LOG_WARN("fail to update location", KR(ret), K(leader));
} else {
LOG_INFO("update location to be follower success", K(leader));
}
} else {
}
if (OB_SUCC(ret)) {
if (!filter_not_readable_replica && OB_FAIL(result.assign(location))) {
LOG_WARN("assign location fail", K(ret), K(location));
} else if (filter_not_readable_replica && OB_FAIL(result.assign_with_only_readable_replica(location))) {
LOG_WARN("assign location with only readable replica fail", K(ret), K(location));
}
}
return ret;
}
int ObPartitionLocationCache::renew_location(const uint64_t table_id, const int64_t partition_id,
const int64_t cluster_id, ObPartitionLocation& location, const int64_t expire_renew_time,
const bool result_filter_not_readable_replica, const bool force_sql_renew, const bool auto_update)
{
int ret = OB_SUCCESS;
ObPartitionLocation new_location;
bool sem_acquired = false;
ObTimeoutCtx ctx;
int64_t abs_timeout_us = -1;
if (!is_inited_) {
ret = OB_NOT_INIT;
LOG_WARN("not init", K(ret));
} else if (!ObIPartitionTable::is_valid_key(table_id, partition_id)) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("invalid argument", K(ret), KT(table_id), K(partition_id));
} else if (OB_FAIL(set_timeout_ctx(ctx))) {
LOG_WARN("failed to set timeout ctx", K(ret));
} else {
abs_timeout_us = ObTimeoutCtx::get_ctx().get_abs_timeout();
}
if (OB_SUCC(ret)) {
// only limit user table fetch location concurrent requests,
// if use table request count is limited, sys table request triggered by user table
// request is limited naturally
// we assume virtual table request is not numerous
if (!is_sys_table(table_id) && !is_virtual_table(table_id)) {
const uint64_t wait_event_time_out = 0;
const int64_t unused = 0;
ObWaitEventGuard wait_guard(ObWaitEventIds::PT_LOCATION_CACHE_LOCK_WAIT,
wait_event_time_out,
static_cast<int64_t>(table_id),
partition_id,
unused);
// ignore acquire fail
int tmp_ret = sem_.acquire(abs_timeout_us);
if (OB_SUCCESS != tmp_ret) {
LOG_WARN("acquire failed", K(tmp_ret));
if (OB_TIMEOUT == tmp_ret) {
ret = OB_TIMEOUT;
LOG_WARN("already timeout", K(ret), K(abs_timeout_us));
}
} else {
sem_acquired = true;
}
}
}
bool refresh_by_rpc = false;
bool refresh_by_sql = false;
bool exist_in_cache = true;
// try to get from cache, maybe other thread has already fetched location
if (OB_SUCC(ret)) {
const bool is_nonblock = false;
const bool filter_not_readable_replica = false;
refresh_by_rpc = false;
if (OB_FAIL(get_from_cache(
table_id, partition_id, cluster_id, is_nonblock, location, filter_not_readable_replica, auto_update))) {
if (OB_ENTRY_NOT_EXIST != ret) {
LOG_WARN("get_from_cache failed", K(table_id), K(partition_id), K(cluster_id), K(ret));
} else {
exist_in_cache = false;
refresh_by_sql = true;
ret = OB_SUCCESS;
}
} else if (!location.is_mark_fail() &&
((!force_sql_renew && location.get_renew_time() > expire_renew_time) ||
(force_sql_renew && location.get_sql_renew_time() > expire_renew_time))) {
refresh_by_sql = false;
} else {
// we need to treat mark_fail location as deleted, must renew it
refresh_by_rpc = cluster_id_ == cluster_id; // standby cluster should renew location by sql
refresh_by_sql = true;
}
}
const int64_t now = ObTimeUtility::current_time();
if (OB_SUCC(ret) && (refresh_by_rpc || refresh_by_sql)) {
EVENT_INC(LOCATION_CACHE_RENEW);
// refresh by rpc
if (OB_SUCC(ret) && refresh_by_rpc) {
bool skip_rpc_renew = false;
if (OB_FAIL(check_skip_rpc_renew_v2(location, skip_rpc_renew))) {
LOG_WARN("check skip rpc renew failed", K(ret), K(location));
ret = OB_SUCCESS;
skip_rpc_renew = true;
}
if (skip_rpc_renew) {
EVENT_INC(LOCATION_CACHE_IGNORE_RPC_RENEW);
}
if (OB_SUCC(ret) && !skip_rpc_renew) {
bool is_new_location_valid = false;
const int64_t rpc_timeout = ctx.get_timeout() / 2; // reserve 50% for fetch from meta table
ctx.set_timeout(rpc_timeout);
if (OB_FAIL(
location_fetcher_.renew_location_with_rpc_v2(nullptr, location, new_location, is_new_location_valid))) {
LOG_WARN("failed to renew location with rpc",
K(ret),
K(table_id),
K(partition_id),
K(location),
K(expire_renew_time),
"cost",
ObTimeUtility::current_time() - now,
K(lbt()));
// we can still fetch location from roottable
ret = OB_SUCCESS;
} else if (is_new_location_valid) {
if (EXECUTE_COUNT_PER_SEC(16)) {
LOG_INFO("fetch location with rpc success",
K(table_id),
K(partition_id),
"old_location",
location,
K(new_location),
K(expire_renew_time),
"cost",
ObTimeUtility::current_time() - now,
K(lbt()));
}
refresh_by_sql = false;
} else {
EVENT_INC(LOCATION_CACHE_RPC_RENEW_FAIL);
LOG_WARN("rpc renew failed, need sql renew",
K(table_id),
K(partition_id),
K(location),
K(expire_renew_time),
"cost",
ObTimeUtility::current_time() - now,
K(lbt()));
}
ctx.set_abs_timeout(abs_timeout_us); // reset timeout
}
} // end rpc renew
// refresh by sql
if (OB_SUCC(ret) && refresh_by_sql) {
if (!auto_update && OB_ALL_CORE_TABLE_TID == extract_pure_id(table_id)) {
// To avoid -4012 error when renew __all_core_table's location cache.
ctx.set_abs_timeout(OB_INVALID_TIMESTAMP);
}
if (OB_FAIL(location_fetcher_.fetch_location(table_id, partition_id, cluster_id, new_location))) {
LOG_WARN("fetch location failed",
K(ret),
KT(table_id),
K(partition_id),
K(expire_renew_time),
"cost",
ObTimeUtility::current_time() - now,
K(lbt()));
if (!auto_update // auto_update is false means renew_location() is running on background worker thread.
&& !exist_in_cache && use_sys_leader_cache(table_id, cluster_id)) {
// When renew location failed and location cache is empty,
// try renew location cache of sys tenant's system tables by RPC.
int tmp_ret = OB_SUCCESS;
ObArray<ObLocationCacheKey> keys;
ObLocationCacheKey key(table_id, partition_id, cluster_id);
if (OB_SUCCESS != (tmp_ret = keys.push_back(key))) {
LOG_WARN("fail to push back key", K(tmp_ret), K(key));
} else if (OB_SUCCESS != (tmp_ret = batch_renew_sys_table_location_by_rpc(keys))) {
LOG_WARN("fail to batch renew sys table location by rpc", K(tmp_ret), K(key));
}
}
} else {
EVENT_INC(LOCATION_CACHE_SQL_RENEW);
}
if (location.get_replica_locations().count() > 0) {
ObTaskController::get().allow_next_syslog();
LOG_INFO("location cache not hit, fetch location by partition_table",
K(ret),
K(location),
K(new_location),
K(expire_renew_time),
"cost",
ObTimeUtility::current_time() - now,
K(lbt()));
}
ctx.set_abs_timeout(abs_timeout_us); // reset timeout
}
if (OB_SUCC(ret)) {
if (OB_FAIL(update_location(table_id, partition_id, cluster_id, new_location))) {
LOG_WARN("update location in cache failed", K(ret), KT(table_id), K(partition_id), K(new_location));
} else if (result_filter_not_readable_replica &&
OB_FAIL(location.assign_with_only_readable_replica(new_location))) {
LOG_WARN("assign with only readable replica fail", K(ret), K(location), K(new_location));
} else if (!result_filter_not_readable_replica && OB_FAIL(location.assign(new_location))) {
LOG_WARN("assign location fail", K(ret), K(location), K(new_location));
}
}
} else {
EVENT_INC(LOCATION_CACHE_RENEW_IGNORED);
}
if (sem_acquired) {
// ignore release fail
int tmp_ret = sem_.release();
if (OB_SUCCESS != tmp_ret) {
LOG_WARN("release failed", K(tmp_ret));
}
}
return ret;
}
/*
* renew location cache by RPC may failed in the following cases:
* 1. location cache is empty.
* 2. there is no avaliable paxos member in local region.
* 3. there is inactive paxos member in local region.
* 4. non cluster private tables in primary cluster before ver 2.2.74.
* 5. cluster is on ofs mode with single zone deployment.
* 5. partition is in physical restore status. TODO: () should be removed not less than ver 3.2.0
*/
int ObPartitionLocationCache::check_skip_rpc_renew_v2(const ObPartitionLocation& location, bool& skip)
{
int ret = OB_SUCCESS;
skip = false;
bool alive = false;
int64_t trace_time = 0;
int64_t paxos_replica_num = 0;
if (!is_inited_) {
ret = OB_NOT_INIT;
LOG_WARN("not init", K(ret));
} else if (location.get_replica_locations().count() <= 0) {
skip = true;
LOG_DEBUG("location count less than 0, skip rpc_renew");
} else if (GCTX.is_standby_cluster() && GET_MIN_CLUSTER_VERSION() < CLUSTER_VERSION_2274 &&
!ObMultiClusterUtil::is_cluster_private_table(location.get_table_id())) {
skip = true;
LOG_DEBUG("slave cluster has no member list, skip rpc_renew");
} else {
skip = false;
bool is_local = false;
FOREACH_CNT_X(l, location.get_replica_locations(), OB_SUCC(ret) && !skip)
{
is_local = false;
if (OB_ISNULL(l)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("replica location is null", K(ret));
} else if (!ObReplicaTypeCheck::is_paxos_replica_V2(l->replica_type_)) {
// nothing todo
} else if (share::ObServerBlacklist::get_instance().is_in_blacklist(
share::ObCascadMember(l->server_, cluster_id_))) {
skip = true;
ObTaskController::get().allow_next_syslog();
LOG_INFO("server is in blacklist, skip rpc renew", K(ret), "server", l->server_);
} else if (OB_FAIL(locality_manager_->is_local_server(l->server_, is_local))) {
LOG_WARN("fail to check is local server", K(ret), "server", l->server_);
ret = OB_SUCCESS;
skip = true;
} else if (!is_local) {
// noting todo
} else if (OB_FAIL(server_tracer_->is_alive(l->server_, alive, trace_time))) {
LOG_WARN("check server alive failed", K(ret), "server", l->server_);
} else if (!alive) {
skip = true;
ObTaskController::get().allow_next_syslog();
LOG_INFO("server is not alive, skip rpc renew", K(ret), K(alive), "server", l->server_, K(is_local));
} else if (l->is_phy_restore_replica()) {
// restore replica refresh location cache by sql
skip = true;
}
if (OB_SUCC(ret) && is_local && ObReplicaTypeCheck::is_paxos_replica_V2(l->replica_type_)) {
paxos_replica_num++;
}
} // end foreach
if (OB_SUCC(ret) && !skip && paxos_replica_num <= 0) {
LOG_DEBUG("paxos replica num less than 1, skip rpc renew", K(paxos_replica_num));
skip = true;
}
}
if (skip) {
LOG_DEBUG("need skip rpc renew", K(location), K(skip));
}
return ret;
}
int ObPartitionLocationCache::update_location(
const uint64_t table_id, const int64_t partition_id, const int64_t cluster_id, const ObPartitionLocation& location)
{
int ret = OB_SUCCESS;
if (!is_inited_) {
ret = OB_NOT_INIT;
LOG_WARN("not init", K(ret));
} else if (!ObIPartitionTable::is_valid_key(table_id, partition_id) || !location.is_valid()) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("invalid argument", K(ret), KT(table_id), K(partition_id), K(location));
} else {
ObLocationCacheKey cache_key(table_id, partition_id, cluster_id);
if (use_sys_cache(table_id)) {
int overwrite_flag = 1;
// try update sys leader cache
if (use_sys_leader_cache(table_id, cluster_id)) {
LocationInfo leader_location_info;
ObReplicaLocation leader_replica;
int tmp_ret = location.get_strong_leader(leader_replica);
if (OB_SUCCESS == tmp_ret) {
leader_location_info.set(leader_replica.server_, location.get_renew_time());
} else {
} // invalid leader location info, no need to set
// set sys leader cache
if (OB_SUCCESS !=
(tmp_ret = sys_leader_cache_.set_refactored(cache_key, leader_location_info, overwrite_flag))) {
LOG_WARN("fail to set sys leader info", KR(tmp_ret), K(cache_key), K(leader_location_info));
} else {
LOG_INFO("renew sys table leader info", KR(tmp_ret), K(cache_key), K(leader_location_info));
}
}
if (OB_FAIL(sys_cache_.set_refactored(cache_key, location, overwrite_flag))) {
LOG_WARN("set new location in sys_cache failed", K(cache_key), K(location), K(ret));
} else {
LOG_TRACE("renew location in sys_cache succeed", K(cache_key), K(location));
}
} else {
// try update leader cache
if (cluster_id_ == cluster_id) {
LocationInfo leader_location_info;
ObReplicaLocation leader_replica;
const bool force_update = false;
int tmp_ret = location.get_strong_leader(leader_replica);
if (OB_SUCCESS == tmp_ret) {
leader_location_info.set(leader_replica.server_, location.get_renew_time());
} else {
} // invalid leader location info, no need to set
// set leader cache
if (OB_SUCCESS !=
(tmp_ret = leader_cache_.set_strong_leader_info(cache_key, leader_location_info, force_update))) {
LOG_WARN("fail to set leader", K(tmp_ret), K(cache_key), K(leader_location_info));
}
}
ObLocationCacheValue cache_value;
char* buffer = NULL;
buffer = static_cast<char*>(ob_malloc(OB_MAX_LOCATION_SERIALIZATION_SIZE, ObModIds::OB_MS_LOCATION_CACHE));
if (NULL == buffer) {
ret = OB_ALLOCATE_MEMORY_FAILED;
} else if (OB_FAIL(location2cache_value(location, buffer, OB_MAX_LOCATION_SERIALIZATION_SIZE, cache_value))) {
LOG_WARN("transform location to cache_value failed", K(location), K(ret));
} else if (OB_FAIL(user_cache_.put(cache_key, cache_value))) {
LOG_WARN("put location to user location cache failed", K(cache_key), K(cache_value), K(ret));
} else {
LOG_TRACE("renew location in user_cache succeed", K(cache_key), K(location));
}
if (NULL != buffer) {
ob_free(buffer);
buffer = NULL;
}
}
}
return ret;
}
int ObPartitionLocationCache::clear_location(
const uint64_t table_id, const int64_t partition_id, const int64_t expire_renew_time, const int64_t cluster_id)
{
int ret = OB_SUCCESS;
if (!is_inited_) {
ret = OB_NOT_INIT;
LOG_WARN("not init", K(ret));
} else if (!ObIPartitionTable::is_valid_key(table_id, partition_id)) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("invalid argument", K(ret), KT(table_id), K(partition_id));
} else {
ObPartitionLocation location;
ObLocationCacheKey cache_key(table_id, partition_id, cluster_id);
if (use_sys_cache(table_id)) {
int hash_ret = sys_cache_.get_refactored(cache_key, location);
if (OB_SUCCESS != hash_ret) {
if (OB_HASH_NOT_EXIST == hash_ret) {
LOG_TRACE("location not exist, clear_location do nothing", K(table_id), K(partition_id));
} else {
ret = hash_ret;
LOG_WARN("get location from sys_cache failed", K(ret), K(cache_key));
}
} else {
if (location.get_renew_time() > expire_renew_time) {
LOG_TRACE("location exist, but renew time is not expired, no need to clear it",
K(table_id),
K(partition_id),
K(location.get_renew_time()),
K(expire_renew_time));
EVENT_INC(LOCATION_CACHE_CLEAR_LOCATION_IGNORED);
} else {
LOG_TRACE("location exist, clear_location mark it fail", K(table_id), K(partition_id));
location.mark_fail();
const int overwrite_flag = 1;
if (OB_FAIL(sys_cache_.set_refactored(cache_key, location, overwrite_flag))) {
LOG_WARN("failed to mark location deleted", K(ret), K(hash_ret), K(cache_key), K(location));
} else {
LOG_TRACE("mark_location_fail in sys_cache success", K(cache_key), K(location));
EVENT_INC(LOCATION_CACHE_CLEAR_LOCATION);
}
}
}
} else {
ObKVCacheHandle handle;
const ObLocationCacheValue* cache_value = NULL;
if (OB_FAIL(user_cache_.get(cache_key, cache_value, handle))) {
if (OB_ENTRY_NOT_EXIST != ret) {
LOG_WARN("failed to get location from user cache", K(ret), K(cache_key));
} else {
ret = OB_SUCCESS;
LOG_TRACE("location not exist, clear_location do nothing", K(table_id), K(partition_id));
}
} else {
ObLocationCacheValue new_cache_value;
char* buffer = NULL;
buffer = static_cast<char*>(ob_malloc(OB_MAX_LOCATION_SERIALIZATION_SIZE, ObModIds::OB_MS_LOCATION_CACHE));
if (NULL == buffer) {
ret = OB_ALLOCATE_MEMORY_FAILED;
} else if (OB_ISNULL(cache_value)) {
ret = OB_ERR_SYS;
LOG_WARN("cache value must not null", K(ret), K(table_id), K(partition_id));
} else if (OB_FAIL(cache_value2location(*cache_value, location))) {
LOG_WARN("failed to transform cache value to location", K(ret), K(cache_value));
} else {
if (location.get_renew_time() > expire_renew_time) {
LOG_TRACE("location exist, but renew time is not expired, no need to clear it",
K(table_id),
K(partition_id),
K(location.get_renew_time()),
K(expire_renew_time));
EVENT_INC(LOCATION_CACHE_CLEAR_LOCATION_IGNORED);
} else {
LOG_TRACE("location exist, clear_location mark it fail", K(table_id), K(partition_id));
location.mark_fail();
if (OB_FAIL(location2cache_value(location, buffer, OB_MAX_LOCATION_SERIALIZATION_SIZE, new_cache_value))) {
LOG_WARN("failed to transform location 2 cache value", K(ret), K(location));
} else if (OB_FAIL(user_cache_.put(cache_key, new_cache_value))) {
LOG_WARN("failed to put new cache value", K(ret), K(cache_key), K(new_cache_value));
} else {
LOG_TRACE("mark_location_fail in user cache success", K(table_id), K(partition_id), K(location));
EVENT_INC(LOCATION_CACHE_CLEAR_LOCATION);
}
}
}
if (NULL != buffer) {
ob_free(buffer);
buffer = NULL;
}
}
}
}
return ret;
}
int ObPartitionLocationCache::vtable_get(const uint64_t table_id, ObIArray<ObPartitionLocation>& locations,
const int64_t expire_renew_time, bool& is_cache_hit)
{
int ret = OB_SUCCESS;
ObSArray<ObPartitionLocation> vtable_locations;
bool need_renew = false;
is_cache_hit = false;
if (!is_inited_) {
ret = OB_NOT_INIT;
LOG_WARN("not init", K(ret));
} else if (OB_INVALID_ID == table_id || !is_virtual_table(table_id)) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("invalid argument", K(ret), KT(table_id));
} else {
ret = get_from_vtable_cache(table_id, vtable_locations);
if (OB_ENTRY_NOT_EXIST == ret) {
ret = OB_SUCCESS;
need_renew = true;
LOG_DEBUG("location not exist, need renew", K(table_id));
} else if (OB_SUCCESS != ret) {
LOG_WARN("get location from vtable cache failed", K(ret), KT(table_id));
} else if (vtable_locations.size() <= 0) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("vtable location size must larger than 0", K(ret), K(vtable_locations));
} else if (vtable_locations.at(0).get_renew_time() <= expire_renew_time) {
need_renew = true;
LOG_DEBUG("location is too old, need renew",
K(table_id),
K(expire_renew_time),
"location renew time",
vtable_locations.at(0).get_renew_time());
} else {
// check server alive
bool alive = false;
int64_t trace_time = 0;
FOREACH_CNT_X(location, vtable_locations, OB_SUCC(ret) && !need_renew)
{
FOREACH_CNT_X(l, location->get_replica_locations(), OB_SUCC(ret) && !need_renew)
{
if (OB_FAIL(server_tracer_->is_alive(l->server_, alive, trace_time))) {
LOG_WARN("check server alive failed", K(ret), "server", l->server_);
} else {
if (!alive && trace_time > location->get_renew_time()) {
need_renew = true;
LOG_DEBUG(
"force renew virtual table location because location on non-alive server", "location", *location);
}
}
}
}
}
if (OB_SUCC(ret)) {
if (need_renew) {
is_cache_hit = false;
} else {
is_cache_hit = true;
}
}
if (OB_SUCC(ret) && need_renew) {
if (OB_FAIL(renew_vtable_location(table_id, vtable_locations))) {
LOG_WARN("renew vtable location failed", KT(table_id), K(ret));
}
}
}
if (OB_SUCC(ret)) {
if (OB_FAIL(locations.assign(vtable_locations))) {
LOG_WARN("assign failed", K(ret));
}
}
return ret;
}
int ObPartitionLocationCache::get_from_vtable_cache(const uint64_t table_id, ObSArray<ObPartitionLocation>& locations)
{
int ret = OB_SUCCESS;
ObKVCacheHandle handle;
ObLocationCacheKey cache_key(table_id, 0, cluster_id_);
const ObLocationCacheValue* cache_value = NULL;
if (!is_inited_) {
ret = OB_NOT_INIT;
LOG_WARN("not init", K(ret));
} else if (OB_INVALID_ID == table_id || !is_virtual_table(table_id)) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("invalid argument", K(ret), KT(table_id));
} else if (OB_FAIL(user_cache_.get(cache_key, cache_value, handle))) {
if (OB_ENTRY_NOT_EXIST != ret) {
LOG_WARN("get location from user cache failed", K(cache_key), K(ret));
} else {
LOG_TRACE("location is not in user cache", K(cache_key), K(ret));
}
} else {
LOG_TRACE("location hit in user cache", K(cache_key), K(ret));
if (OB_FAIL(cache_value2location(*cache_value, locations))) {
LOG_WARN("transform cache value to location failed", K(cache_value), K(ret));
}
}
if (OB_SUCCESS == ret) {
const int64_t now = ObTimeUtility::current_time();
const int64_t first_index = 0;
const int64_t partition_id = 0;
if (locations.size() <= 0) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("locations is empty", K(ret));
} else if (now - locations.at(first_index).get_renew_time() >= config_->virtual_table_location_cache_expire_time) {
int64_t cluster_id = cluster_id_;
ObLocationAsyncUpdateTask task(
*this, table_id, partition_id, now, cluster_id, ObLocationAsyncUpdateTask::MODE_AUTO);
// use temp_ret to avoid overwrite ret
int temp_ret = add_update_task(task);
if (OB_SUCCESS != temp_ret) {
LOG_WARN("add location update task failed", K(locations), K(task), K(temp_ret));
} else {
LOG_DEBUG("add_update_task success", K(task));
}
}
}
return ret;
}
int ObPartitionLocationCache::renew_vtable_location(const uint64_t table_id, ObSArray<ObPartitionLocation>& locations)
{
int ret = OB_SUCCESS;
ObTimeoutCtx ctx;
if (!is_inited_) {
ret = OB_NOT_INIT;
LOG_WARN("not init", K(ret));
} else if (OB_INVALID_ID == table_id || !is_virtual_table(table_id)) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("invalid argument", K(ret), KT(table_id));
} else if (OB_FAIL(set_timeout_ctx(ctx))) {
LOG_WARN("failed to set timeout ctx", K(ret));
} else if (OB_FAIL(location_fetcher_.fetch_vtable_location(table_id, locations))) {
LOG_WARN("fetch_vtable_location failed", KT(table_id), K(ret));
} else if (OB_FAIL(update_vtable_location(table_id, locations))) {
LOG_WARN("update_vtable_location failed", KT(table_id), K(ret));
} else {
ObTaskController::get().allow_next_syslog();
LOG_INFO("renew vtable location success", KT(table_id), K(locations), K(ret));
}
return ret;
}
int ObPartitionLocationCache::update_vtable_location(
const uint64_t table_id, const ObSArray<ObPartitionLocation>& locations)
{
const int64_t MAX_BUFFER_SIZE = 1L << 20; // 1MB
int ret = OB_SUCCESS;
ObLocationCacheKey cache_key(table_id, 0, cluster_id_);
ObLocationCacheValue cache_value;
if (!is_inited_) {
ret = OB_NOT_INIT;
LOG_WARN("not init", K(ret));
} else if (OB_INVALID_ID == table_id || !is_virtual_table(table_id) || locations.empty()) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("invalid argument", K(ret), KT(table_id), "location count", locations.count());
} else {
const int64_t buffer_size = locations.get_serialize_size();
if (buffer_size <= 0 || buffer_size > MAX_BUFFER_SIZE) {
ret = OB_SIZE_OVERFLOW;
LOG_WARN("size overflow", K(ret), K(buffer_size));
} else {
char* buffer = static_cast<char*>(ob_malloc(buffer_size, ObModIds::OB_MS_LOCATION_CACHE));
if (NULL == buffer) {
ret = OB_ALLOCATE_MEMORY_FAILED;
} else if (OB_FAIL(location2cache_value(locations, buffer, buffer_size, cache_value))) {
LOG_WARN("transform location to cache_value failed", K(locations), K(buffer_size), K(ret));
} else if (OB_FAIL(user_cache_.put(cache_key, cache_value))) {
LOG_WARN("put location to user location cache failed", K(cache_key), K(cache_value), K(ret));
} else {
LOG_TRACE("renew location in user_cache succeed", K(cache_key), K(locations));
}
if (NULL != buffer) {
ob_free(buffer);
buffer = NULL;
}
}
}
return ret;
}
int ObPartitionLocationCache::clear_vtable_location(const uint64_t table_id, const int64_t expire_renew_time)
{
int ret = OB_SUCCESS;
ObKVCacheHandle handle;
const ObLocationCacheKey cache_key(table_id, 0, cluster_id_);
const ObLocationCacheValue* cache_value = NULL;
ObSArray<ObPartitionLocation> locations;
if (OB_INVALID_ID == table_id || !is_virtual_table(table_id)) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("invalid table_id", K(ret), KT(table_id), K(expire_renew_time));
} else if (OB_FAIL(user_cache_.get(cache_key, cache_value, handle))) {
if (OB_ENTRY_NOT_EXIST != ret) {
LOG_WARN("failed to get location from user cache", K(ret), K(cache_key));
} else {
ret = OB_SUCCESS;
LOG_TRACE("location not exist, clear_location do nothing", K(table_id));
}
} else {
ObLocationCacheValue new_cache_value;
char* buffer = static_cast<char*>(ob_malloc(OB_MAX_LOCATION_SERIALIZATION_SIZE, ObModIds::OB_MS_LOCATION_CACHE));
if (NULL == buffer) {
ret = OB_ALLOCATE_MEMORY_FAILED;
} else if (OB_ISNULL(cache_value)) {
ret = OB_ERR_SYS;
LOG_WARN("cache value must not null", K(ret), K(table_id));
} else if (OB_FAIL(cache_value2location(*cache_value, locations))) {
LOG_WARN("failed to transform cache value to location", K(ret), K(cache_value));
} else if (locations.size() > 0) {
const ObPartitionLocation& first_location = locations.at(0);
if (first_location.get_renew_time() > expire_renew_time) {
LOG_TRACE("location exist, but renew time is not expired, no need to clear it",
K(table_id),
K(first_location.get_renew_time()),
K(expire_renew_time));
EVENT_INC(LOCATION_CACHE_CLEAR_LOCATION_IGNORED);
} else {
LOG_TRACE("location exist, clear_location mark it fail", K(table_id));
FOREACH_CNT(location, locations)
{
location->mark_fail();
}
if (OB_FAIL(location2cache_value(locations, buffer, OB_MAX_LOCATION_SERIALIZATION_SIZE, new_cache_value))) {
LOG_WARN("failed to transform location 2 cache value", K(ret), K(locations));
} else if (OB_FAIL(user_cache_.put(cache_key, new_cache_value))) {
LOG_WARN("failed to put new cache value", K(ret), K(cache_key), K(new_cache_value));
} else {
LOG_TRACE("mark_location_fail in user cache success", K(table_id), K(locations));
EVENT_INC(LOCATION_CACHE_CLEAR_LOCATION);
}
}
}
if (NULL != buffer) {
ob_free(buffer);
buffer = NULL;
}
}
return ret;
}
int ObPartitionLocationCache::cache_value2location(
const ObLocationCacheValue& cache_value, ObPartitionLocation& location)
{
int ret = OB_SUCCESS;
location.reset();
if (NULL == cache_value.buffer_ || cache_value.size_ <= 0) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("invalid cache_value", "buffer", OB_P(cache_value.buffer_), "size", cache_value.size_, K(ret));
} else {
int64_t pos = 0;
if (OB_SUCC(ret)) {
if (pos + static_cast<int64_t>(sizeof(location.table_id_)) > cache_value.size_) {
ret = OB_DESERIALIZE_ERROR;
LOG_WARN("deserialize table_id failed",
"read pos",
pos + sizeof(location.table_id_),
"buf size",
cache_value.size_,
K(ret));
} else {
MEMCPY(&(location.table_id_), cache_value.buffer_ + pos, sizeof(location.table_id_));
pos += sizeof(location.table_id_);
}
}
if (OB_SUCC(ret)) {
if (pos + static_cast<int64_t>(sizeof(location.partition_id_)) > cache_value.size_) {
ret = OB_DESERIALIZE_ERROR;
LOG_WARN("deserialize partition_id failed",
"read pos",
pos + sizeof(location.partition_id_),
"buf size",
cache_value.size_,
K(ret));
} else {
MEMCPY(&(location.partition_id_), cache_value.buffer_ + pos, sizeof(location.partition_id_));
pos += sizeof(location.partition_id_);
}
}
if (OB_SUCC(ret)) {
if (pos + static_cast<int64_t>(sizeof(location.partition_cnt_)) > cache_value.size_) {
ret = OB_DESERIALIZE_ERROR;
LOG_WARN("deserialize partition_cnt failed",
"read pos",
pos + sizeof(location.partition_cnt_),
"buf size",
cache_value.size_,
K(ret));
} else {
MEMCPY(&(location.partition_cnt_), cache_value.buffer_ + pos, sizeof(location.partition_cnt_));
pos += sizeof(location.partition_cnt_);
}
}
int64_t replica_count = 0;
if (OB_SUCC(ret)) {
if (pos + static_cast<int64_t>(sizeof(replica_count)) > cache_value.size_) {
ret = OB_DESERIALIZE_ERROR;
LOG_WARN("deserialize replica_count failed",
"read pos",
pos + sizeof(replica_count),
"buf size",
cache_value.size_,
K(ret));
} else {
MEMCPY(&replica_count, cache_value.buffer_ + pos, sizeof(replica_count));
pos += sizeof(replica_count);
}
}
if (OB_SUCCESS == ret && replica_count > 0) {
if (pos + static_cast<int64_t>(sizeof(ObReplicaLocation)) * replica_count > cache_value.size_) {
ret = OB_DESERIALIZE_ERROR;
LOG_WARN("deserialize replica_locations failed",
"read pos",
pos + sizeof(ObReplicaLocation) * replica_count,
"buf size",
cache_value.size_,
K(ret));
} else {
ObReplicaLocation replica_location;
for (int64_t i = 0; OB_SUCC(ret) && i < replica_count; ++i) {
if (OB_FAIL(location.replica_locations_.push_back(replica_location))) {
LOG_WARN("push_back failed", K(ret));
}
}
if (OB_SUCC(ret)) {
MEMCPY(&(location.replica_locations_.at(0)),
cache_value.buffer_ + pos,
sizeof(ObReplicaLocation) * replica_count);
pos += sizeof(ObReplicaLocation) * replica_count;
}
}
}
if (OB_SUCC(ret)) {
if (pos + static_cast<int64_t>(sizeof(location.renew_time_)) > cache_value.size_) {
ret = OB_DESERIALIZE_ERROR;
LOG_WARN("deserialize renew_time failed",
"read pos",
pos + sizeof(location.renew_time_),
"buf size",
cache_value.size_,
K(ret));
} else {
MEMCPY(&location.renew_time_, cache_value.buffer_ + pos, sizeof(location.renew_time_));
pos += sizeof(location.renew_time_);
}
}
if (OB_SUCC(ret)) {
if (pos + static_cast<int64_t>(sizeof(location.is_mark_fail_)) > cache_value.size_) {
ret = OB_DESERIALIZE_ERROR;
LOG_WARN("deserialize is_mark_fail_ failed",
"read pos",
pos + sizeof(location.is_mark_fail_),
"buf size",
cache_value.size_,
K(ret));
} else {
MEMCPY(&location.is_mark_fail_, cache_value.buffer_ + pos, sizeof(location.is_mark_fail_));
pos += sizeof(location.is_mark_fail_);
}
}
if (OB_SUCC(ret)) {
if (pos + static_cast<int64_t>(sizeof(location.sql_renew_time_)) > cache_value.size_) {
ret = OB_DESERIALIZE_ERROR;
LOG_WARN("deserialize renew_time failed",
"read pos",
pos + sizeof(location.sql_renew_time_),
"buf size",
cache_value.size_,
K(ret));
} else {
MEMCPY(&location.sql_renew_time_, cache_value.buffer_ + pos, sizeof(location.sql_renew_time_));
pos += sizeof(location.sql_renew_time_);
}
}
}
return ret;
}
int ObPartitionLocationCache::location2cache_value(
const ObPartitionLocation& location, char* buf, const int64_t buf_size, ObLocationCacheValue& cache_value)
{
int ret = OB_SUCCESS;
if (!location.is_valid()) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("invalid location", K(location), K(ret));
} else if (NULL == buf || buf_size <= 0) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("buf is null or invalid buf_size", "buf", OB_P(buf), K(buf_size), K(ret));
} else {
int64_t pos = 0;
const int64_t need_size =
sizeof(location.table_id_) + sizeof(location.partition_id_) + sizeof(location.partition_cnt_) +
sizeof(int64_t) + sizeof(ObReplicaLocation) * location.replica_locations_.count() +
sizeof(location.renew_time_) + sizeof(location.is_mark_fail_) + sizeof(location.sql_renew_time_);
if (need_size > buf_size) {
ret = OB_BUF_NOT_ENOUGH;
LOG_WARN("serialize location failed", K(need_size), K(buf_size), K(ret));
} else {
MEMCPY(buf + pos, &(location.table_id_), sizeof(location.table_id_));
pos += sizeof(location.table_id_);
MEMCPY(buf + pos, &(location.partition_id_), sizeof(location.partition_id_));
pos += sizeof(location.partition_id_);
MEMCPY(buf + pos, &(location.partition_cnt_), sizeof(location.partition_cnt_));
pos += sizeof(location.partition_cnt_);
const int64_t replica_count = location.replica_locations_.count();
MEMCPY(buf + pos, &replica_count, sizeof(replica_count));
pos += sizeof(replica_count);
if (location.replica_locations_.count() > 0) {
MEMCPY(buf + pos, &(location.replica_locations_.at(0)), replica_count * sizeof(ObReplicaLocation));
pos += replica_count * sizeof(ObReplicaLocation);
}
MEMCPY(buf + pos, &(location.renew_time_), sizeof(location.renew_time_));
pos += sizeof(location.renew_time_);
MEMCPY(buf + pos, &(location.is_mark_fail_), sizeof(location.is_mark_fail_));
pos += sizeof(location.is_mark_fail_);
MEMCPY(buf + pos, &(location.sql_renew_time_), sizeof(location.sql_renew_time_));
pos += sizeof(location.sql_renew_time_);
cache_value.buffer_ = buf;
cache_value.size_ = pos;
}
}
return ret;
}
template <typename LOCATION>
int ObPartitionLocationCache::cache_value2location(const ObLocationCacheValue& cache_value, LOCATION& location)
{
int ret = OB_SUCCESS;
NG_TRACE(plc_serialize_begin);
int64_t pos = 0;
if (NULL == cache_value.buffer_ || cache_value.size_ <= 0) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("invalid cache_value", "buffer", OB_P(cache_value.buffer_), "size", cache_value.size_, K(ret));
} else if (OB_FAIL(location.deserialize(cache_value.buffer_, cache_value.size_, pos))) {
LOG_WARN("deserialize location failed", K(cache_value), K(ret));
}
NG_TRACE(plc_serialize_end);
return ret;
}
template <typename LOCATION>
int ObPartitionLocationCache::location2cache_value(
const LOCATION& location, char* buf, const int64_t buf_size, ObLocationCacheValue& cache_value)
{
int ret = OB_SUCCESS;
int64_t pos = 0;
if (NULL == buf || buf_size <= 0) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("buf is null or invalid buf_size", KP(buf), K(buf_size), K(ret));
} else if (OB_FAIL(location.serialize(buf, buf_size, pos))) {
LOG_WARN("serialize location failed", K(location), "buf", reinterpret_cast<int64_t>(buf), K(ret));
} else {
cache_value.size_ = pos;
cache_value.buffer_ = buf;
}
return ret;
}
int ObPartitionLocationCache::add_update_task(const ObLocationAsyncUpdateTask& task)
{
int ret = OB_SUCCESS;
if (!is_inited_) {
ret = OB_NOT_INIT;
LOG_WARN("not init", K(ret));
} else if (!task.is_valid()) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("invalid task", K(ret), K(task));
} else if (task.get_cluster_id() == cluster_id_) {
if (OB_FAIL(local_async_queue_set_.add_task(task))) {
LOG_WARN("fail to add task", K(ret), K(task));
}
} else {
if (OB_FAIL(remote_async_queue_set_.add_task(task))) {
LOG_WARN("fail to add task", K(ret), K(task));
}
}
return ret;
}
int ObPartitionLocationCache::set_timeout_ctx(common::ObTimeoutCtx& ctx)
{
int ret = OB_SUCCESS;
int64_t default_timeout = GCONF.location_cache_refresh_rpc_timeout +
GCONF.location_cache_refresh_sql_timeout;
if (OB_FAIL(ObShareUtil::set_default_timeout_ctx(ctx, default_timeout))) {
LOG_WARN("fail to set default_timeout_ctx", KR(ret));
}
return ret;
}
// link table.
int ObPartitionLocationCache::get_link_table_location(const uint64_t table_id, ObPartitionLocation& location)
{
int ret = OB_SUCCESS;
uint64_t dblink_id = extract_dblink_id(table_id);
ObSchemaGetterGuard schema_guard;
const ObDbLinkSchema* dblink_schema = NULL;
if (OB_FAIL(schema_service_->get_schema_guard(schema_guard))) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("get schema guard failed", K(ret));
} else if (OB_FAIL(schema_guard.get_dblink_schema(dblink_id, dblink_schema))) {
LOG_WARN("get dblink schema failed", K(dblink_id), K(ret));
} else if (OB_ISNULL(dblink_schema)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("NULL ptr", K(dblink_id), K(ret));
} else {
location.set_table_id(table_id);
location.set_partition_id(0);
ObReplicaLocation replica_location;
replica_location.server_ = dblink_schema->get_host_addr();
replica_location.role_ = LEADER;
replica_location.replica_type_ = REPLICA_TYPE_FULL;
replica_location.sql_port_ = dblink_schema->get_host_port();
if (OB_FAIL(location.add(replica_location))) {
LOG_WARN(
"add replica location to partition location failed", KT(table_id), K(replica_location), K(location), K(ret));
}
}
return ret;
}
/*-----batch async renew location-----*/
int ObPartitionLocationCache::batch_process_tasks(
const common::ObIArray<ObLocationAsyncUpdateTask>& tasks, bool& stopped)
{
int ret = OB_SUCCESS;
ObCurTraceId::init(GCONF.self_addr_);
UNUSED(stopped);
if (!is_inited_) {
ret = OB_NOT_INIT;
LOG_WARN("not inited yet", K(ret));
} else if (tasks.count() <= 0) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("invalid task cnt", K(ret));
} else if (OB_FAIL(pre_check(tasks))) {
LOG_WARN("pre_check failed", K(ret), K(tasks));
} else if (tasks.at(0).need_process_alone() && 1 == tasks.count()) {
/*
* In the following cases, we won't renew location by batch.
* 1. renew virtual table's location cache.
* 2. renew __all_core_table's location cache.
* 3. renew core table's location cache.
* P.S: We already deal with compatibility in need_process_alone().
*/
for (int64_t i = 0; i < tasks.count(); i++) { // overwrite ret
const ObLocationAsyncUpdateTask& task = tasks.at(i);
const int64_t now = ObTimeUtility::current_time();
bool force_sql_renew = ObLocationAsyncUpdateTask::MODE_SQL_ONLY == task.get_type();
ObLocationUpdateTask tmp_task(*this,
is_stopped_,
task.get_table_id(),
task.get_partition_id(),
now,
task.get_cluster_id(),
force_sql_renew);
if (OB_FAIL(tmp_task.process())) {
LOG_WARN("fail to process task", K(ret), K(task));
}
}
} else if (OB_FAIL(batch_renew_location(tasks))) {
LOG_WARN("fail to batch renew location", K(ret), K(tasks));
}
return ret;
}
/*
* defence, to make sure tasks in one batch meet the following conditions:
* 1. renew tasks of __all_core_table or core table are not included.
* 2. renew tasks in one bacth has same type and cluster_id.
* 3. renew tasks of system table and virtual table and user table can't be processed in one batch.
*/
int ObPartitionLocationCache::pre_check(const common::ObIArray<ObLocationAsyncUpdateTask>& tasks)
{
int ret = OB_SUCCESS;
if (!is_inited_) {
ret = OB_NOT_INIT;
LOG_WARN("not init", K(ret));
} else if (tasks.count() <= 0) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("invalid task cnt", K(ret));
} else {
for (int64_t i = 0; OB_SUCC(ret) && i < tasks.count(); i++) {
const ObLocationAsyncUpdateTask& cur_task = tasks.at(i);
if (!cur_task.is_valid()) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("invalid task", K(ret), K(cur_task));
} else if (cur_task.need_discard()) {
ret = OB_EAGAIN;
LOG_WARN("cluster type changed, need discard", K(ret), K(cur_task));
} else if (0 == i) {
// do nothing
} else {
// check cluster_id & table_id & type_
const ObLocationAsyncUpdateTask& pre_task = tasks.at(i - 1);
uint64_t cur_table_id = cur_task.get_table_id();
uint64_t pre_table_id = pre_task.get_table_id();
if (cur_task.get_cluster_id() != pre_task.get_cluster_id()) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("unexpected cluster_id", K(ret), K(cur_task), K(pre_task));
} else if (cur_task.get_type() != cur_task.get_type()) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("unexpected type", K(ret), K(cur_task), K(pre_task));
} else if (cur_task.need_process_alone()) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("task need process alone, unexpected table_id", K(ret), K(cur_task), K(pre_task));
} else if (is_sys_table(cur_table_id) != is_sys_table(pre_table_id)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("unexpected table_id", K(ret), K(cur_task), K(pre_task));
} else if (!is_inner_table(cur_table_id) &&
extract_tenant_id(cur_table_id) != extract_tenant_id(pre_table_id)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("unexpected table_id", K(ret), K(cur_task), K(pre_task));
} else {
// For now, we ensure renew task of __all_core_table(or core table) will process alone.
}
}
}
}
return ret;
}
int ObPartitionLocationCache::batch_renew_location(const common::ObIArray<ObLocationAsyncUpdateTask>& tasks)
{
int ret = OB_SUCCESS;
if (!is_inited_) {
ret = OB_NOT_INIT;
LOG_WARN("not init", K(ret));
} else if (tasks.count() <= 0) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("invalid task cnt", K(ret));
} else {
const int64_t start = ObTimeUtility::current_time();
const int64_t expire_renew_time = start - GCONF.location_cache_refresh_min_interval;
int64_t succ_cnt = 0;
int64_t wait_cost = 0;
if (OB_SUCC(ret)) {
for (int64_t i = 0; i < tasks.count(); i++) { // overwrite ret
wait_cost += (start - tasks.at(i).get_add_timestamp());
}
}
// Step 1 : try acquire lock
bool sem_acquired = false;
int64_t abs_timeout_us = -1;
if (OB_SUCC(ret)) {
// only limit user table fetch location concurrent requests,
// if use table request count is limited, sys table request triggered by user table
// request is limited naturally
// we assume virtual table request is not numerous
uint64_t table_id = tasks.at(0).get_table_id();
if (!is_sys_table(table_id) && !is_virtual_table(table_id)) {
const uint64_t wait_event_time_out = 0;
const int64_t unused = 0;
ObWaitEventGuard wait_guard(
ObWaitEventIds::PT_LOCATION_CACHE_LOCK_WAIT, wait_event_time_out, unused, unused, unused);
// ignore acquire fail
abs_timeout_us = ObTimeUtility::current_time() + GCONF.location_cache_refresh_rpc_timeout +
GCONF.location_cache_refresh_sql_timeout;
int tmp_ret = sem_.acquire(abs_timeout_us);
if (OB_SUCCESS != tmp_ret) {
LOG_WARN("acquire failed", K(tmp_ret));
if (OB_TIMEOUT == tmp_ret) {
ret = OB_TIMEOUT;
LOG_WARN("already timeout", K(ret), K(abs_timeout_us));
}
} else {
sem_acquired = true;
}
}
}
// Step 2 : filter tasks
ObLocationAsyncUpdateTask::Type renew_type = tasks.at(0).get_type();
int64_t cluster_id = tasks.at(0).get_cluster_id();
common::ObArenaAllocator allocator("RenewLocation");
ObSEArray<ObPartitionLocation*, UNIQ_TASK_QUEUE_BATCH_EXECUTE_NUM> locations;
if (cluster_id != cluster_id_) { // renew location by SQL when fetch locations across clusters.
renew_type = ObLocationAsyncUpdateTask::MODE_SQL_ONLY;
}
if (OB_SUCC(ret)) {
bool force_sql_renew = ObLocationAsyncUpdateTask::MODE_SQL_ONLY == renew_type;
const bool is_nonblock = false;
const bool filter_not_readable_replica = false;
ObPartitionLocation* location = NULL;
for (int64_t i = 0; i < tasks.count(); i++) { // overwrite ret
const ObLocationAsyncUpdateTask& task = tasks.at(i);
location = NULL;
uint64_t table_id = task.get_table_id();
int64_t partition_id = task.get_partition_id();
bool need_renew = false;
bool auto_update = false;
if (OB_FAIL(ObPartitionLocation::alloc_new_location(allocator, location))) {
LOG_WARN("fail to alloc location", K(ret), K(task));
} else if (OB_FAIL(get_from_cache(table_id,
partition_id,
cluster_id,
is_nonblock,
*location,
filter_not_readable_replica,
auto_update))) {
if (OB_ENTRY_NOT_EXIST != ret) {
LOG_WARN("get_from_cache failed", K(table_id), K(partition_id), K(cluster_id), K(ret));
} else {
location->set_table_id(table_id);
location->set_partition_id(partition_id);
need_renew = true;
ret = OB_SUCCESS;
}
} else if (!location->is_mark_fail() &&
((!force_sql_renew && location->get_renew_time() > expire_renew_time) ||
(force_sql_renew && location->get_sql_renew_time() > expire_renew_time))) {
need_renew = false;
succ_cnt++;
EVENT_INC(LOCATION_CACHE_RENEW_IGNORED);
} else {
need_renew = true;
}
if (OB_SUCC(ret) && need_renew) {
if (OB_FAIL(locations.push_back(location))) {
LOG_WARN("fail to push back task", K(ret), K(task));
}
}
}
ret = OB_SUCCESS;
}
// Step 3 : batch renew
if (OB_SUCC(ret) && locations.count() > 0) {
ObSEArray<ObPartitionLocation*, UNIQ_TASK_QUEUE_BATCH_EXECUTE_NUM> new_locations;
common::ObTimeoutCtx ctx;
if (OB_FAIL(set_batch_timeout_ctx(locations.count(), renew_type, ctx))) {
LOG_WARN("fail to set time ctx", K(ret));
} else if (ObLocationAsyncUpdateTask::MODE_SQL_ONLY == renew_type) {
// renew location by sql
ObSEArray<common::ObPartitionKey, UNIQ_TASK_QUEUE_BATCH_EXECUTE_NUM> keys;
for (int64_t i = 0; OB_SUCC(ret) && i < locations.count(); i++) {
ObPartitionLocation*& location = locations.at(i);
if (OB_ISNULL(location)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("location is null", K(ret));
} else {
ObPartitionKey key(location->get_table_id(), location->get_partition_id(), 0);
if (OB_FAIL(keys.push_back(key))) {
LOG_WARN("fail to push back key", K(ret), K(key));
}
}
}
if (OB_FAIL(ret)) {
} else if (OB_FAIL(location_fetcher_.batch_fetch_location(keys, cluster_id, allocator, new_locations))) {
LOG_WARN("batch fetch location failed",
K(ret),
K(expire_renew_time),
"task_cnt",
locations.count(),
"cost",
ObTimeUtility::current_time() - start);
// When renew location failed and location cache is empty,
// try renew location cache of sys tenant's system tables by RPC.
int tmp_ret = OB_SUCCESS;
ObArray<ObLocationCacheKey> renew_keys;
for (int64_t i = 0; OB_SUCCESS == tmp_ret && i < locations.count(); i++) {
ObPartitionLocation*& location = locations.at(i);
if (OB_ISNULL(location)) {
tmp_ret = OB_ERR_UNEXPECTED;
LOG_WARN("location is null", K(tmp_ret));
} else if (0 == location->get_replica_locations().count() &&
use_sys_leader_cache(location->get_table_id(), cluster_id)) {
ObLocationCacheKey key(location->get_table_id(), location->get_partition_id(), cluster_id);
if (OB_FAIL(renew_keys.push_back(key))) {
LOG_WARN("fail to push back key", K(ret), K(key));
}
}
}
if (OB_SUCCESS != tmp_ret) {
} else if (renew_keys.count() <= 0) {
// skip
} else if (OB_SUCCESS != (tmp_ret = batch_renew_sys_table_location_by_rpc(renew_keys))) {
LOG_WARN("fail to batch renew sys table location by rpc", K(tmp_ret), "key_cnt", renew_keys.count());
}
} else {
EVENT_INC(LOCATION_CACHE_SQL_RENEW);
}
} else {
// renew location by rpc
if (OB_FAIL(batch_renew_location_by_rpc(locations, allocator, new_locations))) {
LOG_WARN("fail to batch renew location by rpc", K(ret));
} else {
LOG_INFO("location cache not hit, batch fetch location by rpc",
K(ret),
"location_cnt",
locations.count(),
"new_location_cnt",
new_locations.count());
}
}
// update cache
if (OB_SUCC(ret)) {
for (int64_t i = 0; i < new_locations.count(); i++) { // overwrite ret
ObPartitionLocation*& new_location = new_locations.at(i);
if (OB_ISNULL(new_location)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("location is null", K(ret), K(i));
} else if (OB_FAIL(update_location(
new_location->get_table_id(), new_location->get_partition_id(), cluster_id, *new_location))) {
LOG_WARN("fail to update location", K(ret), KPC(new_location));
}
if (OB_SUCC(ret)) {
succ_cnt++;
LOG_DEBUG("renew location succ", K(ret), K(renew_type), KPC(new_location));
}
}
ret = OB_SUCCESS;
}
// always release locations and new_locations
for (int64_t i = 0; i < locations.count(); i++) {
ObPartitionLocation*& location = locations.at(i);
if (OB_NOT_NULL(location)) {
location->~ObPartitionLocation();
location = NULL;
}
}
for (int64_t i = 0; i < new_locations.count(); i++) {
ObPartitionLocation*& location = new_locations.at(i);
if (OB_NOT_NULL(location)) {
location->~ObPartitionLocation();
location = NULL;
}
}
}
// Step 4 : try release lock
if (sem_acquired) {
// ignore release fail
int tmp_ret = sem_.release();
if (OB_SUCCESS != tmp_ret) {
LOG_WARN("release failed", K(tmp_ret));
}
}
// Step 5 : calc statistic
const int64_t end = ObTimeUtility::current_time();
auto* statistics = GET_TSI(TSILocationCacheStatistics);
if (OB_ISNULL(statistics)) {
LOG_WARN("fail to get statistic", "ret", OB_ERR_UNEXPECTED);
} else {
bool sql_renew = (ObLocationAsyncUpdateTask::MODE_SQL_ONLY == renew_type);
ObLocationCacheQueueSet::Type type =
ObLocationCacheQueueSet::get_queue_type(tasks.at(0).get_table_id(), tasks.at(0).get_partition_id());
(void)statistics->calc(type, sql_renew, succ_cnt, tasks.count() - succ_cnt, wait_cost, end - start);
const int64_t interval = 1 * 1000 * 1000; // 1s
if (TC_REACH_TIME_INTERVAL(interval)) {
(void)statistics->dump();
(void)statistics->reset();
}
}
}
return ret;
}
int ObPartitionLocationCache::batch_renew_location_by_rpc(const common::ObIArray<ObPartitionLocation*>& locations,
common::ObIAllocator& allocator, common::ObIArray<ObPartitionLocation*>& new_locations)
{
int ret = OB_SUCCESS;
if (locations.count() <= 0) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("invalid location cnt", K(ret));
} else {
ObSEArray<const ObPartitionLocation*, UNIQ_TASK_QUEUE_BATCH_EXECUTE_NUM> rpc_locations;
// filter location should renew by sql
for (int64_t i = 0; OB_SUCC(ret) && i < locations.count(); i++) {
bool skip_rpc_renew = false;
const ObPartitionLocation* location = locations.at(i);
if (OB_ISNULL(location)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("location is null", K(ret));
} else if (OB_FAIL(check_skip_rpc_renew_v2(*location, skip_rpc_renew))) {
LOG_WARN("check skip rpc renew failed", K(ret), K(location));
ret = OB_SUCCESS;
skip_rpc_renew = true;
}
if (skip_rpc_renew) {
EVENT_INC(LOCATION_CACHE_IGNORE_RPC_RENEW);
}
if (OB_SUCC(ret) && !skip_rpc_renew) {
if (OB_FAIL(rpc_locations.push_back(location))) {
LOG_WARN("push back location failed", K(ret), KPC(location));
}
}
}
// renew location by rpc
if (OB_SUCC(ret)) {
if (OB_FAIL(location_fetcher_.batch_renew_location_with_rpc(rpc_locations, allocator, new_locations))) {
LOG_WARN("fail to batch renew location by rpc", K(ret), "partition_cnt", rpc_locations.count());
}
}
// partition should renew location by sql
if (OB_SUCC(ret) && locations.count() != new_locations.count()) {
ObHashSet<ObPartitionKey> key_set;
if (OB_FAIL(key_set.create(UNIQ_TASK_QUEUE_BATCH_EXECUTE_NUM))) {
LOG_WARN("fail to create set", K(ret));
}
for (int64_t i = 0; OB_SUCC(ret) && i < new_locations.count(); i++) {
ObPartitionLocation*& new_location = new_locations.at(i);
if (OB_ISNULL(new_location)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("location is null", K(ret));
} else if (OB_FAIL(key_set.set_refactored(
ObPartitionKey(new_location->get_table_id(), new_location->get_partition_id(), 0)))) {
LOG_WARN("fail to set hash set", K(ret), KPC(new_location));
}
}
for (int64_t i = 0; OB_SUCC(ret) && i < locations.count(); i++) {
const ObPartitionLocation* location = locations.at(i);
if (OB_ISNULL(location)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("location is null", K(ret));
} else {
ObPartitionKey key(location->get_table_id(), location->get_partition_id(), 0);
int hash_ret = key_set.exist_refactored(key);
if (OB_HASH_EXIST == hash_ret) {
// do nothing
} else if (OB_HASH_NOT_EXIST == hash_ret) {
ObLocationAsyncUpdateTask task(*this,
key.get_table_id(),
key.get_partition_id(),
ObTimeUtility::current_time(),
cluster_id_,
ObLocationAsyncUpdateTask::MODE_SQL_ONLY);
int temp_ret = add_update_task(task);
if (OB_SUCCESS != temp_ret) {
LOG_WARN("add location update task failed", K(temp_ret), KPC(location), K(task));
} else {
/* Should renew location by SQL in the following cases:
* 1.check_skip_rpc_renew_v2() shows we should skip renew location by rpc.
* 2.rpc renew failed
* - 2.1. Call RPC on observer failed.
* - 2.2. Partial partitions' in one RPC return failure.
* - 2.3. Partial partitions' new locations not matched.(member_list or non paxos member maybe changed)
*/
LOG_DEBUG("should renew location by sql, add_update_task success", K(temp_ret), K(task));
}
} else {
ret = OB_SUCC(ret) ? OB_ERR_UNEXPECTED : hash_ret;
LOG_WARN("check key exist failed", K(ret));
}
}
}
}
}
return ret;
}
int ObPartitionLocationCache::process_barrier(const ObLocationAsyncUpdateTask& task, bool& stopped)
{
UNUSEDx(task, stopped);
return OB_NOT_SUPPORTED;
}
int ObPartitionLocationCache::set_batch_timeout_ctx(
const int64_t task_cnt, ObLocationAsyncUpdateTask::Type type, common::ObTimeoutCtx& ctx)
{
int ret = OB_SUCCESS;
int64_t abs_timeout_us = ctx.get_abs_timeout();
UNUSED(task_cnt);
if (abs_timeout_us < 0) {
if (ObLocationAsyncUpdateTask::MODE_SQL_ONLY == type) {
abs_timeout_us = ObTimeUtility::current_time() + 2 * GCONF.location_cache_refresh_sql_timeout;
} else {
abs_timeout_us = ObTimeUtility::current_time() + 2 * GCONF.location_cache_refresh_rpc_timeout;
}
}
if (THIS_WORKER.get_timeout_ts() > 0 && THIS_WORKER.get_timeout_ts() < abs_timeout_us) {
abs_timeout_us = THIS_WORKER.get_timeout_ts();
}
if (OB_FAIL(ctx.set_abs_timeout(abs_timeout_us))) {
LOG_WARN("set timeout failed", K(ret), K(abs_timeout_us));
} else if (ctx.is_timeouted()) {
ret = OB_TIMEOUT;
LOG_WARN("is timeout",
K(ret),
"abs_timeout",
ctx.get_abs_timeout(),
"this worker timeout ts",
THIS_WORKER.get_timeout_ts());
}
return ret;
}
/*-----batch async renew location end -----*/
bool ObPartitionLocationCache::use_sys_cache(const uint64_t table_id) const
{
return OB_SYS_TENANT_ID == extract_tenant_id(table_id) && is_sys_table(table_id);
}
bool ObPartitionLocationCache::use_sys_leader_cache(const uint64_t table_id, const int64_t cluster_id) const
{
return OB_SYS_TENANT_ID == extract_tenant_id(table_id) && OB_ALL_CORE_TABLE_TID != extract_pure_id(table_id) &&
is_sys_table(table_id) && cluster_id == cluster_id_;
}
int ObPartitionLocationCache::batch_renew_sys_table_location_by_rpc(common::ObIArray<ObLocationCacheKey>& keys)
{
int ret = OB_SUCCESS;
ObTimeoutCtx ctx;
ctx.set_timeout(GCONF.location_cache_refresh_rpc_timeout);
ObPartitionLocation core_table_location;
bool is_nonblock = false;
bool filter_not_readable_replica = true;
bool auto_update = false;
ObArray<ObLocationLeader> results;
if (OB_FAIL(get_from_cache(combine_id(OB_SYS_TENANT_ID, OB_ALL_CORE_TABLE_TID),
ObIPartitionTable::ALL_CORE_TABLE_PARTITION_ID,
cluster_id_,
is_nonblock,
core_table_location,
filter_not_readable_replica,
auto_update))) {
if (OB_ENTRY_NOT_EXIST != ret) {
LOG_WARN("fail to get from cache", K(ret));
} else {
ret = OB_SUCCESS; // overwrite ret
}
} else if (OB_FAIL(location_fetcher_.batch_renew_sys_table_location_by_rpc(core_table_location, keys, results))) {
LOG_WARN("fail to batch renew sys table location by rpc", K(ret), "keys_cnt", keys.count());
} else {
int overwrite_flag = 1;
for (int64_t i = 0; OB_SUCC(ret) && i < results.count(); i++) {
const ObLocationCacheKey& key = results.at(i).get_key();
const LocationInfo& leader_info = results.at(i).get_strong_leader_info();
if (OB_FAIL(sys_leader_cache_.set_refactored(key, leader_info, overwrite_flag))) {
LOG_WARN("fail to set sys table leader cache", K(ret), "keys_cnt", keys.count());
} else {
ObTaskController::get().allow_next_syslog();
LOG_INFO("renew sys table leader by rpc", K(ret), K(key), K(leader_info));
}
}
}
return ret;
}
int64_t ObPartitionLocationCache::get_primary_cluster_id() const
{
int64_t cluster_id = common::OB_INVALID_ID;
if (OB_NOT_NULL(remote_server_provider_)) {
cluster_id = remote_server_provider_->get_primary_cluster_id();
}
return cluster_id;
}
} // end namespace share
} // end namespace oceanbase
Reference in New Issue View Git Blame Copy Permalink