Fix backup check disk is enough bug

deps/oblib/src/lib/utility/ob_tracepoint.h

@@ -446,6 +446,7 @@ public:
     EN_BACKUP_BACKUPSET_DO_SCHEDULE = 184,
     EN_BACKUP_CHECK_BACKUP_POINT_EXIST = 185,
     EN_STOP_TENANT_LOG_ARCHIVE_BACKUP = 186,
+    EN_BACKUP_SERVER_DISK_IS_FULL = 188,
     // 下面请从201开始
     EN_CHECK_STANDBY_CLUSTER_SCHEMA_CONDITION = 201,
     EN_ALLOCATE_LOB_BUF_FAILED = 202,

src/rootserver/ob_root_backup.cpp

@@ -671,6 +671,8 @@ int ObRootBackup::do_sys_tenant_backup(
   } else {
     if (OB_FAIL(add_backup_info_lock(info, updater, info_manager))) {
       LOG_WARN("failed to add backup info lock", K(ret), K(info));
+    } else if (OB_FAIL(check_server_disk_stat(info, updater.get_trans()))) {
+      LOG_WARN("failed to check server disk stat", K(ret), K(info));
     } else if (OB_FAIL(check_tenants_backup_task_failed(info, info_manager, updater.get_trans()))) {
       LOG_WARN("failed to check tenants backup task failed", K(ret), K(info));
     } else if (OB_FAIL(do_with_all_finished_info(info, updater, info_manager))) {
@@ -3123,6 +3125,80 @@ int ObRootBackup::do_tenant_update_task_his_and_backup_set_file(
   return ret;
 }
 
+int ObRootBackup::check_server_disk_stat(const ObBaseBackupInfoStruct &info, common::ObISQLClient &sys_tenant_trans)
+{
+  int ret = OB_SUCCESS;
+  ObTenantBackupTaskInfo sys_task_info;
+  ObTenantBackupTaskInfo sys_dest_task_info;
+  const ObZone zone;
+  ObArray<share::ObServerStatus> server_status_array;
+  const int64_t NO_LIMIT_PERCENT = 100;
+
+  if (!is_inited_) {
+    ret = OB_NOT_INIT;
+    LOG_WARN("root backup do not init", K(ret));
+  } else if (!info.is_valid()) {
+    ret = OB_INVALID_ARGUMENT;
+    LOG_WARN("check server disk stat get invalid argument", K(ret), K(info));
+  } else if (OB_FAIL(get_tenant_backup_task(
+                 info.tenant_id_, info.backup_set_id_, info.incarnation_, sys_tenant_trans, sys_task_info))) {
+    LOG_WARN("failed to get tenant backup task", K(ret), K(info));
+  } else if (!sys_task_info.is_result_succeed()) {
+    // do nothing
+  } else if (OB_FAIL(server_mgr_->get_server_statuses(zone, server_status_array))) {
+    LOG_WARN("failed to get server statuses", K(ret));
+  } else {
+    for (int64_t i = 0; OB_SUCC(ret) && i < server_status_array.count(); ++i) {
+      const ObServerStatus &status = server_status_array.at(i);
+      if (!status.is_active() || !status.in_service()) {
+        // do nothing
+      } else if (0 == status.resource_info_.disk_total_) {
+      } else {
+        const int64_t data_disk_usage_limit_percentage = GCONF.data_disk_usage_limit_percentage;
+        int64_t used_percent = (100 * status.resource_info_.disk_in_use_) / status.resource_info_.disk_total_;
+
+#ifdef ERRSIM
+        if (OB_SUCC(ret)) {
+          ret = E(EventTable::EN_BACKUP_SERVER_DISK_IS_FULL) OB_SUCCESS;
+          if (OB_FAIL(ret)) {
+            STORAGE_LOG(ERROR, "fake EN_BACKUP_SERVER_DISK_IS_FULL", K(ret));
+            used_percent = 100;
+            ret = OB_SUCCESS;
+          }
+        }
+#endif
+
+        if (data_disk_usage_limit_percentage != NO_LIMIT_PERCENT && used_percent >= data_disk_usage_limit_percentage) {
+          FLOG_INFO("server disk is almost full", K(ret), K(used_percent), K(status));
+          sys_dest_task_info = sys_task_info;
+          sys_dest_task_info.result_ = OB_CS_OUTOF_DISK_SPACE;
+          if (OB_FAIL(update_tenant_backup_task(sys_tenant_trans, sys_task_info, sys_dest_task_info))) {
+            LOG_WARN("failed to update tenant backup task", K(ret), K(sys_task_info), K(sys_dest_task_info));
+          } else {
+            ROOTSERVICE_EVENT_ADD("backup",
+                "backup server disk is almost full",
+                "server",
+                status.server_,
+                "disk_in_use",
+                status.resource_info_.disk_in_use_,
+                "disk_total",
+                status.resource_info_.disk_total_,
+                "result",
+                OB_CS_OUTOF_DISK_SPACE);
+            break;
+          }
+        }
+      }
+    }
+  }
+  return ret;
+}
+
+bool ObRootBackup::is_force_cancel_() const
+{
+  return GCONF.backup_dest.get_value_string().empty();
+}
+
 ObTenantBackup::ObTenantBackup()
     : is_inited_(false),
       schema_service_(NULL),

src/rootserver/ob_root_backup.h

@@ -253,6 +253,10 @@ private:
   int insert_tenant_backup_set_file_failed(ObMySQLTransaction& trans, const share::ObTenantBackupTaskInfo& task_info);
   int do_tenant_update_task_his_and_backup_set_file(
       const share::ObTenantBackupTaskInfo& task_info, const ObBackupSetFileInfo& backup_set_file_info);
+  int check_server_disk_stat(const ObBaseBackupInfoStruct &info, common::ObISQLClient &sys_tenant_trans);
+
+private:
+  bool is_force_cancel_() const;
 
 private:
   bool is_inited_;