From ede121344628ff0afcf158c42466153df41b7495 Mon Sep 17 00:00:00 2001
From: yy <yy0@oceanbase.com>
Date: Wed, 16 Jun 2021 11:53:07 +0800
Subject: [PATCH] force drop tenant when physical restore fail

---
 .../restore/ob_restore_scheduler.cpp          | 39 ++++++++++++++++++-
 src/rootserver/restore/ob_restore_scheduler.h |  2 +
 src/share/parameter/ob_parameter_seed.ipp     |  4 ++
 3 files changed, 43 insertions(+), 2 deletions(-)

diff --git a/src/rootserver/restore/ob_restore_scheduler.cpp b/src/rootserver/restore/ob_restore_scheduler.cpp
index cf06db391..b2fdc241a 100644
--- a/src/rootserver/restore/ob_restore_scheduler.cpp
+++ b/src/rootserver/restore/ob_restore_scheduler.cpp
@@ -2347,6 +2347,8 @@ int ObRestoreScheduler::restore_fail(const ObPhysicalRestoreJob& job_info)
     LOG_WARN("not init", K(ret));
   } else if (OB_FAIL(check_stop())) {
     LOG_WARN("restore scheduler stopped", K(ret));
+  } else if (OB_FAIL(drop_tenant_force_if_necessary(job_info))) {
+    LOG_WARN("failed to drop_tenant_force_if_necessary", K(ret), K(job_info));
   } else if (OB_FAIL(restore_op.init(sql_proxy_))) {
     LOG_WARN("fail init", K(ret));
   } else if (OB_FAIL(restore_op.recycle_job(job_info.job_id_, PHYSICAL_RESTORE_FAIL))) {
@@ -2757,5 +2759,38 @@ int ObRestoreScheduler::do_upgrade_post(const ObPhysicalRestoreJob& job_info)
   return ret;
 }
 
-}  // end namespace rootserver
-}  // end namespace oceanbase
+int ObRestoreScheduler::drop_tenant_force_if_necessary(const ObPhysicalRestoreJob &job_info)
+{
+  int ret = OB_SUCCESS;
+  const bool need_force_drop = GCONF._auto_drop_tenant_if_restore_failed;
+  if (!inited_) {
+    ret = OB_NOT_INIT;
+    LOG_WARN("not inited", K(ret));
+  } else if (OB_FAIL(check_stop())) {
+    LOG_WARN("restore scheduler stopped", K(ret));
+  } else if (need_force_drop) {
+    obrpc::ObDropTenantArg arg;
+    arg.exec_tenant_id_ = OB_SYS_TENANT_ID;
+    arg.tenant_name_ = job_info.tenant_name_;
+    arg.if_exist_ = true;
+    arg.delay_to_drop_ = false;
+    ObSqlString sql;
+    const int64_t TIMEOUT_PER_RPC = GCONF.rpc_timeout; // default 2s
+    const int64_t DEFAULT_TIMEOUT = 10 * 1000 * 1000L; // 10s
+    int64_t rpc_timeout = max(TIMEOUT_PER_RPC, DEFAULT_TIMEOUT);
+    if (OB_FAIL(sql.append_fmt("DROP TENANT IF EXISTS %s FORCE", arg.tenant_name_.ptr()))) {
+      LOG_WARN("fail to generate sql", K(ret), K(arg));
+    } else if (FALSE_IT(arg.ddl_stmt_str_ = sql.string())) {
+    } else if (OB_FAIL(rpc_proxy_->timeout(rpc_timeout).drop_tenant(arg))) {
+      LOG_WARN("fail to drop tenant", K(ret), K(arg));
+    } else {
+      LOG_INFO("drop_tenant_force after restore fail", K(job_info));
+    }
+  } else {
+    LOG_INFO("no need to drop tenant after restore fail", K(job_info));
+  }
+  return ret;
+}
+
+} // end namespace rootserver
+} // end namespace oceanbase
diff --git a/src/rootserver/restore/ob_restore_scheduler.h b/src/rootserver/restore/ob_restore_scheduler.h
index e0757369d..09bbfa73a 100644
--- a/src/rootserver/restore/ob_restore_scheduler.h
+++ b/src/rootserver/restore/ob_restore_scheduler.h
@@ -165,6 +165,8 @@ class ObRestoreScheduler : public ObRsReentrantThread, public share::ObCheckStop
   void record_rs_event(const share::ObPhysicalRestoreJob& job, const share::PhysicalRestoreStatus next_status);
   share::PhysicalRestoreStatus get_next_status(int return_ret, share::PhysicalRestoreStatus current_status);
 
+  private:
+  int drop_tenant_force_if_necessary(const share::ObPhysicalRestoreJob &job_info);
   private:
   bool inited_;
   mutable ObRestoreIdling idling_;
diff --git a/src/share/parameter/ob_parameter_seed.ipp b/src/share/parameter/ob_parameter_seed.ipp
index 8a248c495..83631b6a5 100644
--- a/src/share/parameter/ob_parameter_seed.ipp
+++ b/src/share/parameter/ob_parameter_seed.ipp
@@ -1430,3 +1430,7 @@ DEF_TIME(ilog_index_expire_time, OB_CLUSTER_PARAMETER, "7d", "[0s, 60d]",
     "specifies the expire time of ilog_index, can use this parameter to limit the"
     "memory usage of file_id_cache",
     ObParameterAttr(Section::CLOG, Source::DEFAULT, EditLevel::DYNAMIC_EFFECTIVE));
+//auto drop restoring tenant if physical restore fails
+DEF_BOOL(_auto_drop_tenant_if_restore_failed, OB_CLUSTER_PARAMETER, "True",
+         "auto drop restoring tenant if physical restore fails",
+         ObParameterAttr(Section::OBSERVER, Source::DEFAULT, EditLevel::DYNAMIC_EFFECTIVE));