add server event for unit gc

src/observer/omt/ob_multi_tenant.cpp

@@ -138,6 +138,7 @@
 #endif
 #include "observer/table/ob_htable_lock_mgr.h"
 #include "observer/table/ob_table_session_pool.h"
+#include "observer/ob_server_event_history_table_operator.h"
 
 using namespace oceanbase;
 using namespace oceanbase::lib;
@@ -1762,44 +1763,56 @@ int ObMultiTenant::del_tenant(const uint64_t tenant_id)
     ret = OB_ERR_UNEXPECTED;
     LOG_WARN("hidden tenant can't be deleted", K(ret), K(tenant_id));
   } else  {
+    const ObUnitInfoGetter::ObTenantConfig local_unit = tenant->get_unit();
+    const ObUnitInfoGetter::ObUnitStatus local_unit_status = local_unit.unit_status_;
+    // add a event when try to gc for the first time
+    if (local_unit_status != ObUnitInfoGetter::ObUnitStatus::UNIT_WAIT_GC_IN_OBSERVER &&
+        local_unit_status != ObUnitInfoGetter::ObUnitStatus::UNIT_DELETING_IN_OBSERVER) {
+      SERVER_EVENT_ADD("unit", "start unit gc", "tenant_id", tenant_id,
+          "unit_id", local_unit.unit_id_, "unit_status", "DELETING");
+    }
+
     // Ensure to write delete_tenant_prepare_slog only once
-    ObUnitInfoGetter::ObUnitStatus old_unit_status = tenant->get_unit_status();
+    if (local_unit_status != ObUnitInfoGetter::UNIT_DELETING_IN_OBSERVER) {
-    if (old_unit_status != ObUnitInfoGetter::UNIT_DELETING_IN_OBSERVER) {
       tenant->set_unit_status(ObUnitInfoGetter::UNIT_DELETING_IN_OBSERVER);
       tenant->set_create_status(ObTenantCreateStatus::DELETING);
       if (OB_FAIL(write_delete_tenant_prepare_slog(tenant_id))) {
-        LOG_WARN("fail to write delete tenant slog", K(ret), K(tenant_id), K(old_unit_status));
+        LOG_WARN("fail to write delete tenant slog", K(ret), K(tenant_id), K(local_unit_status));
-        tenant->set_unit_status(old_unit_status);
+        tenant->set_unit_status(local_unit_status);
       }
     }
-  }
 
-  if (OB_SUCC(ret)) {
-    do {
-      // 保证remove_tenant, clear_persistent_data可以幂等重试,
-      // 如果失败会但不是加锁失败会一直无限重试, 保证如果prepare log写成功一定会有commit日志，
-      // 即使这个过程中宕机重启, 重启回放日志时会继续删除并且补一条delete commit log
-      bool remove_tenant_succ = false;
-      if (OB_FAIL(remove_tenant(tenant_id, remove_tenant_succ))) {
-        LOG_WARN("fail to remove tenant", K(ret), K(tenant_id));
-        // If lock failed, the tenant is not removed from tenants_list,
-        // Here can break and leave ObTenantNodeBalancer::check_del_tenant to retry again,
-        // in this case, the deletion of other tenants does not get stuck.
-        // Otherwise it will have to retry indefinitely here, because the tenant cannot be obtained
-        if (false == remove_tenant_succ) {
-          break;
-        } else {
-          SLEEP(1);
-        }
-      } else if (OB_FAIL(clear_persistent_data(tenant_id))) {
-        LOG_ERROR("fail to clear persistent_data", K(ret), K(tenant_id));
-        SLEEP(1);
-      } else if (OB_FAIL(write_delete_tenant_commit_slog(tenant_id))) {
-        LOG_WARN("fail to write delete tenant commit slog", K(ret), K(tenant_id));
-      }
-    } while (OB_FAIL(ret));
     if (OB_SUCC(ret)) {
-      lib::ObMallocAllocator::get_instance()->recycle_tenant_allocator(tenant_id);
+      do {
+        // 保证remove_tenant, clear_persistent_data可以幂等重试,
+        // 如果失败会但不是加锁失败会一直无限重试, 保证如果prepare log写成功一定会有commit日志，
+        // 即使这个过程中宕机重启, 重启回放日志时会继续删除并且补一条delete commit log
+        bool remove_tenant_succ = false;
+        if (OB_FAIL(remove_tenant(tenant_id, remove_tenant_succ))) {
+          LOG_WARN("fail to remove tenant", K(ret), K(tenant_id));
+          // If lock failed, the tenant is not removed from tenants_list,
+          // Here can break and leave ObTenantNodeBalancer::check_del_tenant to retry again,
+          // in this case, the deletion of other tenants does not get stuck.
+          // Otherwise it will have to retry indefinitely here, because the tenant cannot be obtained
+          if (false == remove_tenant_succ) {
+            break;
+          } else {
+            SLEEP(1);
+          }
+        } else if (OB_FAIL(clear_persistent_data(tenant_id))) {
+          LOG_ERROR("fail to clear persistent_data", K(ret), K(tenant_id));
+          SLEEP(1);
+        } else if (OB_FAIL(write_delete_tenant_commit_slog(tenant_id))) {
+          LOG_WARN("fail to write delete tenant commit slog", K(ret), K(tenant_id));
+        }
+      } while (OB_FAIL(ret));
+
+      if (OB_SUCC(ret)) {
+        lib::ObMallocAllocator::get_instance()->recycle_tenant_allocator(tenant_id);
+        // add a event when finish gc unit
+        SERVER_EVENT_ADD("unit", "finish unit gc", "tenant_id", tenant_id,
+            "unit_id", local_unit.unit_id_, "unit_status", "DELETED");
+      }
     }
   }
 

src/observer/omt/ob_tenant_node_balancer.cpp

@@ -35,6 +35,7 @@
 #include "storage/tx_storage/ob_ls_service.h"
 #include "storage/meta_mem/ob_tenant_meta_mem_mgr.h"
 #include "storage/slog_ckpt/ob_server_checkpoint_slog_handler.h"
+#include "observer/ob_server_event_history_table_operator.h"
 #ifdef OB_BUILD_TDE_SECURITY
 #include "share/ob_master_key_getter.h"
 #endif
@@ -458,6 +459,7 @@ int ObTenantNodeBalancer::fetch_effective_tenants(const TenantUnits &old_tenants
   for (int64_t i = 0; OB_SUCC(ret) && i < old_tenants.count(); i++) {
     found = false;
     const ObUnitInfoGetter::ObTenantConfig &tenant_config = old_tenants.at(i);
+    const ObUnitInfoGetter::ObUnitStatus local_unit_status = tenant_config.unit_status_;
     for (int64_t j = 0; j < new_tenants.count(); j++) {
       if (tenant_config.tenant_id_ == new_tenants.at(j).tenant_id_) {
         new_tenants.at(j).create_timestamp_ = tenant_config.create_timestamp_;
@@ -499,19 +501,24 @@ int ObTenantNodeBalancer::fetch_effective_tenants(const TenantUnits &old_tenants
             // need wait gc in observer
             // NOTE: only update unit status when can not release resource
             if (!is_released) {
-              tenants.at(tenants.count() - 1).unit_status_ =
+              tenants.at(tenants.count() - 1).unit_status_ = ObUnitInfoGetter::UNIT_WAIT_GC_IN_OBSERVER;
-                  ObUnitInfoGetter::UNIT_WAIT_GC_IN_OBSERVER;
+              // add a event when try to gc for the first time
+              if (local_unit_status != ObUnitInfoGetter::ObUnitStatus::UNIT_WAIT_GC_IN_OBSERVER &&
+                  local_unit_status != ObUnitInfoGetter::ObUnitStatus::UNIT_DELETING_IN_OBSERVER) {
+                SERVER_EVENT_ADD("unit", "start unit gc", "tenant_id", tenant_config.tenant_id_,
+                    "unit_id", tenant_config.unit_id_, "unit_status", "WAIT GC");
+              }
             }
 
             LOG_INFO("[DELETE_TENANT] tenant has been dropped. can not delete tenant",
-                K(is_released),
+                K(is_released), "local_unit_status", ObUnitInfoGetter::get_unit_status_str(local_unit_status),
                 "is_removed", tenant_config.is_removed_,
                 "create_timestamp", tenant_config.create_timestamp_,
                 K(life_time), K(tenant_config));
           }
         } else {
             LOG_INFO("[DELETE_TENANT] tenant has been dropped. can delete tenant",
-                K(is_released),
+                K(is_released), "local_unit_status", ObUnitInfoGetter::get_unit_status_str(local_unit_status),
                 "is_removed", tenant_config.is_removed_,
                 "create_timestamp", tenant_config.create_timestamp_,
                 K(life_time), K(tenant_config));