From 9a61edbf3ad05ea6b164e5665047a7ec7b1a44e2 Mon Sep 17 00:00:00 2001
From: hezuojiao <hezuojiao@gmail.com>
Date: Thu, 8 Feb 2024 08:21:59 +0000
Subject: [PATCH] [CP] Modify cache leader of auto increment to tenant
 isolation

---
 src/share/ob_gais_client.cpp | 43 ++++++++++++++++++++++--------------
 src/share/ob_gais_client.h   | 14 ++++--------
 2 files changed, 30 insertions(+), 27 deletions(-)

diff --git a/src/share/ob_gais_client.cpp b/src/share/ob_gais_client.cpp
index 90b8808708..e2a52adc68 100644
--- a/src/share/ob_gais_client.cpp
+++ b/src/share/ob_gais_client.cpp
@@ -31,12 +31,16 @@ namespace share
 int ObGAISClient::init(const ObAddr &self, ObGAISRequestRpc *gais_request_rpc)
 {
   int ret = OB_SUCCESS;
+  ObMemAttr attr(OB_SERVER_TENANT_ID, ObModIds::OB_AUTOINCREMENT);
+  SET_USE_500(attr);
   if (OB_UNLIKELY(is_inited_)) {
     ret = OB_INIT_TWICE;
     LOG_WARN("init twice", KR(ret));
   } else if (OB_UNLIKELY(!self.is_valid()) || OB_ISNULL(gais_request_rpc)) {
     ret = OB_INVALID_ARGUMENT;
     LOG_WARN("invalid argument", KR(ret), K(self), KP(gais_request_rpc));
+  } else if (OB_FAIL(gais_cache_leader_map_.create(32 /*init leader size*/, attr, attr))) {
+    LOG_WARN("fail to init leader map", K(ret));
   } else {
     self_ = self;
     gais_request_rpc_ = gais_request_rpc;
@@ -51,7 +55,7 @@ void ObGAISClient::reset()
   is_inited_ = false;
   self_.reset();
   gais_request_rpc_ = NULL;
-  reset_cache_leader_();
+  gais_cache_leader_map_.clear();
 }
 
 int ObGAISClient::get_value(const AutoincKey &key,
@@ -275,22 +279,27 @@ int ObGAISClient::clear_global_autoinc_cache(const AutoincKey &key)
 
 int ObGAISClient::get_leader_(const uint64_t tenant_id, ObAddr &leader)
 {
-  int ret = OB_SUCCESS;
-  const int64_t cluster_id = GCONF.cluster_id;
-  lib::ObMutexGuard guard(cache_leader_mutex_);
-  if (OB_LIKELY(gais_cache_leader_.is_valid())) {
-    leader = gais_cache_leader_;
-  } else if (OB_ISNULL(GCTX.location_service_)) {
-    ret = OB_NOT_INIT;
-    LOG_WARN("location cache is NULL", K(ret));
-  } else if (OB_FAIL(GCTX.location_service_->nonblock_get_leader(
-                                             cluster_id, tenant_id, GAIS_LS, leader))) {
-    LOG_WARN("gais nonblock get leader failed", K(ret), K(tenant_id), K(GAIS_LS));
-  } else if (OB_UNLIKELY(!leader.is_valid())) {
-    ret = OB_ERR_UNEXPECTED;
-    LOG_WARN("get invaild lear from location adapter", K(ret), K(leader));
+  int ret = gais_cache_leader_map_.get_refactored(tenant_id, leader);
+  if (OB_SUCC(ret)) {
+  } else if (ret == OB_HASH_NOT_EXIST) {
+    ret = OB_SUCCESS;
+    const int64_t cluster_id = GCONF.cluster_id;
+    if (OB_ISNULL(GCTX.location_service_)) {
+      ret = OB_NOT_INIT;
+      LOG_WARN("location cache is NULL", K(ret));
+    } else if (OB_FAIL(GCTX.location_service_->nonblock_get_leader(
+                                              cluster_id, tenant_id, GAIS_LS, leader))) {
+      LOG_WARN("gais nonblock get leader failed", K(ret), K(tenant_id), K(GAIS_LS));
+    } else if (OB_UNLIKELY(!leader.is_valid())) {
+      ret = OB_ERR_UNEXPECTED;
+      LOG_WARN("get invaild lear from location adapter", K(ret), K(leader));
+    } else if (OB_FAIL(gais_cache_leader_map_.set_refactored(tenant_id, leader, 1))) {
+      LOG_WARN("fail to set leader to map", K(ret), K(tenant_id), K(leader));
+    } else {
+      LOG_INFO("succ to refresh leader", K(cluster_id), K(tenant_id), K(leader));
+    }
   } else {
-    gais_cache_leader_ = leader;
+    LOG_WARN("fail get cache from hash map", K(ret));
   }
   return ret;
 }
@@ -299,7 +308,7 @@ int ObGAISClient::refresh_location_(const uint64_t tenant_id)
 {
   int ret = OB_SUCCESS;
   const int64_t cluster_id = GCONF.cluster_id;
-  reset_cache_leader_();
+  gais_cache_leader_map_.erase_refactored(tenant_id);
   if (OB_ISNULL(GCTX.location_service_)) {
     ret = OB_NOT_INIT;
     LOG_WARN("location cache is NULL", K(ret));
diff --git a/src/share/ob_gais_client.h b/src/share/ob_gais_client.h
index 3c6289b912..76ea2cc402 100644
--- a/src/share/ob_gais_client.h
+++ b/src/share/ob_gais_client.h
@@ -25,12 +25,12 @@ namespace share
 class ObGAISClient
 {
 public:
-  ObGAISClient() : is_inited_(false), self_(), gais_request_rpc_(nullptr), gais_cache_leader_(),
-  cache_leader_mutex_(common::ObLatchIds::AUTO_INCREMENT_LEADER_LOCK) { }
+  ObGAISClient() : is_inited_(false), self_(), gais_request_rpc_(nullptr), gais_cache_leader_map_()
+  { }
   ~ObGAISClient() { }
   int init(const common::ObAddr &self, share::ObGAISRequestRpc *gais_request_rpc);
   void reset();
-  TO_STRING_KV(K_(self), K_(gais_cache_leader));
+  TO_STRING_KV(K_(self), "map_size", gais_cache_leader_map_.size());
 
 public:
   int get_value(const AutoincKey &key,
@@ -59,18 +59,12 @@ public:
 private:
   int get_leader_(const uint64_t tenant_id, common::ObAddr &leader);
   int refresh_location_(const uint64_t tenant_id);
-  inline void reset_cache_leader_()
-  {
-    lib::ObMutexGuard guard(cache_leader_mutex_);
-    gais_cache_leader_.reset();
-  }
 
 private:
   bool is_inited_;
   common::ObAddr self_;
   share::ObGAISRequestRpc *gais_request_rpc_;
-  common::ObAddr gais_cache_leader_;
-  lib::ObMutex cache_leader_mutex_;
+  common::hash::ObHashMap<uint64_t, common::ObAddr> gais_cache_leader_map_;
 };
 
 } // share