fix tenant level backup restore failure

src/share/backup/ob_backup_info_mgr.cpp

@@ -1142,60 +1142,25 @@ int ObRestoreBackupInfoUtil::get_restore_backup_info_v2_(
     const GetRestoreBackupInfoParam &param, ObRestoreBackupInfo &info)
 {
   int ret = OB_SUCCESS;
-  ObFakeBackupLeaseService fake_backup_lease;
-  ObExternSingleBackupSetInfoMgr backup_info_mgr;
-  ObLogArchiveBackupInfoMgr log_archive_backup_info_mgr;
-  ObExternTenantLocalityInfoMgr tenant_locality_info_mgr;
-  ObExternPGListMgr pg_list_mgr;
-  ObSimpleBackupSetPath simple_set_path;      // largest backup set path
-  ObSimpleBackupPiecePath simple_piece_path;  // smallest backup piece path
-  ObBackupPath piece_backup_path;
   const char *backup_dest = param.backup_dest_;
   const char *backup_cluster_name = param.backup_cluster_name_;
   const int64_t cluster_id = param.cluster_id_;
   const int64_t incarnation = param.incarnation_;
   const int64_t restore_timestamp = param.restore_timestamp_;
-  const char *passwd_array = param.passwd_array_;
-  const int64_t cluster_version = ObClusterVersion::get_instance().get_cluster_version();
-
-  ObExternLogArchiveBackupInfo log_archive_backup_info;
-  ObTenantLogArchiveStatus log_archive_status;
-  ObExternTenantLocalityInfo tenant_locality_info;
   ObBackupPieceInfo piece_info;
   ObBackupSetFileInfo backup_set_info;
-
+  ObExternTenantLocalityInfo tenant_locality_info;
-  if (OB_FAIL(param.get_smallest_backup_piece_path(simple_piece_path))) {
+  bool is_snapshot_restore = false;
-    LOG_WARN("failed to get smallest simple backup piece path", K(ret));
+  if (OB_FAIL(
-  } else if (OB_FAIL(param.get_largest_backup_set_path(simple_set_path))) {
+          inner_get_restore_backup_set_info_(param, backup_set_info, tenant_locality_info, info.sys_pg_key_list_))) {
-    LOG_WARN("failed to get smallest largest backup set path", K(ret));
+    LOG_WARN("failed to inner get restore backup set info", K(ret));
-  } else if (OB_FAIL(backup_info_mgr.init(simple_set_path, fake_backup_lease))) {
+  } else if (OB_FAIL(check_is_snapshot_restore_(param, backup_set_info, is_snapshot_restore))) {
-    LOG_WARN("failed to init backup info mgr", K(ret), K(simple_set_path));
+    LOG_WARN("failed to check is snapshot backup", K(ret), K(backup_set_info));
-  } else if (OB_FAIL(backup_info_mgr.get_extern_backup_set_file_info(passwd_array, backup_set_info))) {
+  } else if (!is_snapshot_restore && OB_FAIL(inner_get_restore_backup_piece_info_(param, piece_info))) {
-    LOG_WARN("failed to find backup info", K(ret), K(restore_timestamp));
+    LOG_WARN("failed to inner get restore backup piece info", K(ret));
-  } else if (OB_FAIL(piece_backup_path.init(simple_piece_path.get_simple_path()))) {
+  } else if (!is_snapshot_restore && backup_set_info.snapshot_version_ < piece_info.start_ts_) {
-    LOG_WARN("failed to init piece backup path", K(ret));
-  } else if (OB_FAIL(piece_backup_path.join(OB_STR_TENANT_CLOG_SINGLE_BACKUP_PIECE_INFO))) {
-    LOG_WARN("failed to join single backup piece info");
-  } else if (OB_FAIL(log_archive_backup_info_mgr.read_external_single_backup_piece_info(
-                 piece_backup_path, simple_piece_path.get_storage_info(), piece_info, fake_backup_lease))) {
-    LOG_WARN("failed to read external single backup piece info", K(ret), K(piece_backup_path), K(simple_piece_path));
-  } else if (backup_set_info.snapshot_version_ < piece_info.start_ts_) {
     ret = OB_ISOLATED_BACKUP_SET;
     LOG_WARN("log archive status is not continues with backup info", K(ret), K(backup_set_info));
-  } else if (OB_FAIL(tenant_locality_info_mgr.init(simple_set_path, fake_backup_lease))) {
-    LOG_WARN("failed to init tenant locality info mgr", K(ret), K(simple_set_path));
-  } else if (OB_FAIL(tenant_locality_info_mgr.get_extern_tenant_locality_info(tenant_locality_info))) {
-    LOG_WARN("failed to find tenant locality info", K(ret));
-  } else if (backup_set_info.cluster_version_ > cluster_version) {
-    ret = OB_NOT_SUPPORTED;
-    LOG_ERROR("cannot restore from newer cluster to older cluster", K(ret), K(cluster_version), K(backup_set_info));
-  } else if (OB_FAIL(pg_list_mgr.init(simple_set_path, fake_backup_lease))) {
-    LOG_WARN("failed to get sys pg list", K(ret), K(backup_set_info), K(simple_set_path));
-  } else if (OB_FAIL(pg_list_mgr.get_sys_pg_list(info.sys_pg_key_list_))) {
-    LOG_WARN("failed to get sys pg list", K(ret), K(backup_set_info));
-  } else if (info.sys_pg_key_list_.empty()) {
-    ret = OB_ERR_UNEXPECTED;
-    LOG_WARN("sys pg key list should not be empty", K(ret), K(backup_set_info));
   } else {
     info.compat_mode_ = tenant_locality_info.compat_mode_;
     info.frozen_data_version_ = backup_set_info.backup_data_version_;
@@ -1214,15 +1179,86 @@ int ObRestoreBackupInfoUtil::get_restore_backup_info_v2_(
     info.physical_restore_info_.inc_backup_set_id_ = backup_set_info.backup_set_id_;
     info.physical_restore_info_.incarnation_ = incarnation;
     info.physical_restore_info_.tenant_id_ = tenant_locality_info.tenant_id_;
-    info.physical_restore_info_.log_archive_round_ = piece_info.key_.round_id_;
     info.physical_restore_info_.compatible_ = backup_set_info.compatible_;
     info.physical_restore_info_.cluster_version_ = backup_set_info.cluster_version_;
     info.physical_restore_info_.backup_date_ = backup_set_info.date_;
+    if (!is_snapshot_restore) {
+      info.physical_restore_info_.log_archive_round_ = piece_info.key_.round_id_;
+    }
     FLOG_INFO("get_restore_backup_info", K(info), K(piece_info), K(backup_set_info));
   }
   return ret;
 }
 
+int ObRestoreBackupInfoUtil::check_is_snapshot_restore_(
+    const GetRestoreBackupInfoParam &param, const ObBackupSetFileInfo &backup_set_info, bool &is_snapshot_restore)
+{
+  int ret = OB_SUCCESS;
+  is_snapshot_restore = false;
+  if (param.backup_piece_path_list_.empty() && !param.backup_set_path_list_.empty()) {
+    is_snapshot_restore = backup_set_info.snapshot_version_ == param.restore_timestamp_;
+    LOG_INFO("check is snapshot backup", K(backup_set_info.snapshot_version_), K(param.restore_timestamp_));
+  }
+  return ret;
+}
+
+int ObRestoreBackupInfoUtil::inner_get_restore_backup_set_info_(const GetRestoreBackupInfoParam &param,
+    ObBackupSetFileInfo &backup_set_info, ObExternTenantLocalityInfo &tenant_locality_info,
+    common::ObIArray<common::ObPGKey> &sys_pg_key_list)
+{
+  int ret = OB_SUCCESS;
+  ObFakeBackupLeaseService fake_backup_lease;
+  ObExternPGListMgr pg_list_mgr;
+  ObExternSingleBackupSetInfoMgr backup_info_mgr;
+  ObExternTenantLocalityInfoMgr tenant_locality_info_mgr;
+  const char *passwd_array = param.passwd_array_;
+  const int64_t cluster_version = ObClusterVersion::get_instance().get_cluster_version();
+  ObSimpleBackupSetPath simple_set_path;  // largest backup set path
+  if (OB_FAIL(param.get_largest_backup_set_path(simple_set_path))) {
+    LOG_WARN("failed to get smallest largest backup set path", K(ret));
+  } else if (OB_FAIL(backup_info_mgr.init(simple_set_path, fake_backup_lease))) {
+    LOG_WARN("failed to init backup info mgr", K(ret), K(simple_set_path));
+  } else if (OB_FAIL(backup_info_mgr.get_extern_backup_set_file_info(passwd_array, backup_set_info))) {
+    LOG_WARN("failed to find backup info", K(ret), K(passwd_array));
+  } else if (OB_FAIL(tenant_locality_info_mgr.init(simple_set_path, fake_backup_lease))) {
+    LOG_WARN("failed to init tenant locality info mgr", K(ret), K(simple_set_path));
+  } else if (OB_FAIL(tenant_locality_info_mgr.get_extern_tenant_locality_info(tenant_locality_info))) {
+    LOG_WARN("failed to find tenant locality info", K(ret));
+  } else if (backup_set_info.cluster_version_ > cluster_version) {
+    ret = OB_NOT_SUPPORTED;
+    LOG_ERROR("cannot restore from newer cluster to older cluster", K(ret), K(cluster_version), K(backup_set_info));
+  } else if (OB_FAIL(pg_list_mgr.init(simple_set_path, fake_backup_lease))) {
+    LOG_WARN("failed to get sys pg list", K(ret), K(backup_set_info), K(simple_set_path));
+  } else if (OB_FAIL(pg_list_mgr.get_sys_pg_list(sys_pg_key_list))) {
+    LOG_WARN("failed to get sys pg list", K(ret), K(backup_set_info));
+  } else if (sys_pg_key_list.empty()) {
+    ret = OB_ERR_UNEXPECTED;
+    LOG_WARN("sys pg key list should not be empty", K(ret), K(backup_set_info));
+  }
+  return ret;
+}
+
+int ObRestoreBackupInfoUtil::inner_get_restore_backup_piece_info_(
+    const GetRestoreBackupInfoParam &param, ObBackupPieceInfo &piece_info)
+{
+  int ret = OB_SUCCESS;
+  ObFakeBackupLeaseService fake_backup_lease;
+  ObLogArchiveBackupInfoMgr log_archive_backup_info_mgr;
+  ObSimpleBackupPiecePath simple_piece_path;  // smallest backup piece path
+  ObBackupPath piece_backup_path;
+  if (OB_FAIL(param.get_smallest_backup_piece_path(simple_piece_path))) {
+    LOG_WARN("failed to get smallest simple backup piece path", K(ret));
+  } else if (OB_FAIL(piece_backup_path.init(simple_piece_path.get_simple_path()))) {
+    LOG_WARN("failed to init piece backup path", K(ret));
+  } else if (OB_FAIL(piece_backup_path.join(OB_STR_TENANT_CLOG_SINGLE_BACKUP_PIECE_INFO))) {
+    LOG_WARN("failed to join single backup piece info");
+  } else if (OB_FAIL(log_archive_backup_info_mgr.read_external_single_backup_piece_info(
+                 piece_backup_path, simple_piece_path.get_storage_info(), piece_info, fake_backup_lease))) {
+    LOG_WARN("failed to read external single backup piece info", K(ret), K(piece_backup_path), K(simple_piece_path));
+  }
+  return ret;
+}
+
 int ObRestoreBackupInfoUtil::get_restore_sys_table_ids(
     const ObPhysicalRestoreInfo &info, common::ObIArray<common::ObPartitionKey> &pkey_list)
 {

src/share/backup/ob_backup_info_mgr.h

@@ -45,7 +45,6 @@ struct ObLogArchiveSimpleInfo final {
       K_(cur_piece_create_date), K_(is_piece_freezing), K_(prev_piece_id), K_(prev_piece_create_date));
 };
 
-// 用于切主后确认pg的checkpoint ts，可能会触发内部表的访问。
 class ObLogArchiveInfoMgr final {
 public:
   static ObLogArchiveInfoMgr &get_instance();
@@ -171,6 +170,13 @@ private:
   static int get_restore_backup_info_v1_(const GetRestoreBackupInfoParam &param, ObRestoreBackupInfo &info);
   // get info from simple path level
   static int get_restore_backup_info_v2_(const GetRestoreBackupInfoParam &param, ObRestoreBackupInfo &info);
+  static int check_is_snapshot_restore_(
+      const GetRestoreBackupInfoParam &param, const ObBackupSetFileInfo &backup_set_info, bool &is_snapshot_restore);
+  static int inner_get_restore_backup_set_info_(const GetRestoreBackupInfoParam &param,
+      ObBackupSetFileInfo &backup_set_info, ObExternTenantLocalityInfo &tenant_locality_info,
+      common::ObIArray<common::ObPGKey> &sys_pg_keys);
+  static int inner_get_restore_backup_piece_info_(
+      const GetRestoreBackupInfoParam &param, ObBackupPieceInfo &piece_info);
 };
 
 class ObRestoreFatalErrorReporter : public share::ObThreadPool {

src/share/backup/ob_multi_backup_dest_util.cpp

@@ -157,6 +157,7 @@ int ObMultiBackupDestUtil::get_multi_backup_path_list(const bool is_preview, con
       int64_t snapshot_version = 0;
       int64_t start_replay_log_ts = 0;
       bool is_compat_path = false;
+      bool is_snapshot_restore = false;
       if (OB_FAIL(get_backup_set_list(is_preview,
               cluster_name,
               cluster_id,
@@ -166,8 +167,11 @@ int ObMultiBackupDestUtil::get_multi_backup_path_list(const bool is_preview, con
               set_list,
               snapshot_version,
               start_replay_log_ts,
-              is_compat_path))) {
+              is_compat_path,
+              is_snapshot_restore))) {
         LOG_WARN("failed to get backup set", KR(ret), K(tenant_id), K(restore_timestamp), K(list));
+      } else if (is_snapshot_restore) {
+        LOG_INFO("tenant level backup no need get piece list", K(tenant_id));
       } else {
         if (is_compat_path) {
           if (OB_FAIL(get_compat_backup_piece_list(cluster_name,
@@ -675,7 +679,7 @@ int ObMultiBackupDestUtil::get_cluster_backup_dest(const ObBackupDest &backup_de
 int ObMultiBackupDestUtil::get_backup_set_list(const bool is_preview, const char *cluster_name,
     const int64_t cluster_id, const uint64_t tenant_id, const int64_t restore_timestamp,
     const common::ObString &backup_dest_str, common::ObArray<ObSimpleBackupSetPath> &list, int64_t &snapshot_version,
-    int64_t &start_replay_log_ts, bool &is_compat_path)
+    int64_t &start_replay_log_ts, bool &is_compat_path, bool &is_snapshot_restore)
 {
   int ret = OB_SUCCESS;
   snapshot_version = 0;
@@ -703,10 +707,13 @@ int ObMultiBackupDestUtil::get_backup_set_list(const bool is_preview, const char
                  tmp_list,
                  snapshot_version,
                  start_replay_log_ts,
-                 is_compat_path))) {
+                 is_compat_path,
+                 is_snapshot_restore))) {
     LOG_WARN("failed to inner get backup set list", KR(ret), K(tenant_id), K(backup_dest));
   } else if (OB_FAIL(append(list, tmp_list))) {
     LOG_WARN("failed to add array", KR(ret), K(tmp_list));
+  } else {
+    LOG_INFO("get backup set list", K(list));
   }
   return ret;
 }
@@ -714,7 +721,7 @@ int ObMultiBackupDestUtil::get_backup_set_list(const bool is_preview, const char
 int ObMultiBackupDestUtil::do_get_backup_set_list(const bool is_preview, const char *cluster_name,
     const int64_t cluster_id, const uint64_t tenant_id, const int64_t restore_timestamp,
     const ObBackupDest &backup_dest, common::ObArray<ObSimpleBackupSetPath> &path_list, int64_t &snapshot_version,
-    int64_t &start_replay_log_ts, bool &is_compat_path)
+    int64_t &start_replay_log_ts, bool &is_compat_path, bool &is_snapshot_restore)
 {
   int ret = OB_SUCCESS;
   path_list.reset();
@@ -744,7 +751,8 @@ int ObMultiBackupDestUtil::do_get_backup_set_list(const bool is_preview, const c
                      backup_dest,
                      path_list,
                      snapshot_version,
-                     start_replay_log_ts))) {
+                     start_replay_log_ts,
+                     is_snapshot_restore))) {
         LOG_WARN("failed to do get cluster level backup set list",
             KR(ret),
             K(is_preview),
@@ -768,6 +776,8 @@ int ObMultiBackupDestUtil::do_get_backup_set_list(const bool is_preview, const c
         } else {
           LOG_WARN("failed to get extern backup set file infos", KR(ret), K(simple_path));
         }
+      } else if (OB_FAIL(check_is_snapshot_restore(info, restore_timestamp, is_snapshot_restore))) {
+        LOG_WARN("failed to check is snapshot backup", K(ret), K(info));
       } else {
         simple_path.backup_set_id_ = info.backup_set_id_;
         simple_path.copy_id_ = info.copy_id_;
@@ -787,7 +797,7 @@ int ObMultiBackupDestUtil::do_get_backup_set_list(const bool is_preview, const c
 int ObMultiBackupDestUtil::do_get_backup_set_list_from_cluster_level(const bool is_preview, const char *cluster_name,
     const int64_t cluster_id, const uint64_t tenant_id, const int64_t restore_timestamp,
     const ObBackupDest &backup_dest, common::ObArray<ObSimpleBackupSetPath> &path_list, int64_t &snapshot_version,
-    int64_t &start_replay_log_ts)
+    int64_t &start_replay_log_ts, bool &is_snapshot_restore)
 {
   int ret = OB_SUCCESS;
   ObClusterBackupDest cluster_backup_dest;
@@ -823,7 +833,8 @@ int ObMultiBackupDestUtil::do_get_backup_set_list_from_cluster_level(const bool
                  file_infos,
                  path_list,
                  snapshot_version,
-                 start_replay_log_ts))) {
+                 start_replay_log_ts,
+                 is_snapshot_restore))) {
     LOG_WARN("failed to do inner get backup set list", KR(ret), K(restore_timestamp));
   } else {
     std::sort(path_list.begin(), path_list.end(), cmp_set);
@@ -869,8 +880,10 @@ int ObMultiBackupDestUtil::do_get_backup_set_list_from_cluster_level(const bool
               ObClusterBackupDest cluster_backup_dest;
               ObBackupDest tmp_backup_dest;
               char tmp_simple_path_str[OB_MAX_BACKUP_DEST_LENGTH] = "";
-              if (OB_FAIL(cluster_backup_dest.set(
+              if (OB_FAIL(check_is_snapshot_restore(tmp_file_info, restore_timestamp, is_snapshot_restore))) {
-                      tmp_file_info.backup_dest_.ptr(), cluster_name, cluster_id, OB_START_INCARNATION))) {
+                LOG_WARN("failed to check is snaptshot backup", K(ret), K(tmp_file_info));
+              } else if (OB_FAIL(cluster_backup_dest.set(
+                             tmp_file_info.backup_dest_.ptr(), cluster_name, cluster_id, OB_START_INCARNATION))) {
                 LOG_WARN("failed to set cluster backup dest");
               } else if (OB_FAIL(base_data_path_info.set(cluster_backup_dest,
                              tmp_file_info.tenant_id_,
@@ -908,7 +921,8 @@ int ObMultiBackupDestUtil::do_get_backup_set_list_from_cluster_level(const bool
 
 int ObMultiBackupDestUtil::do_inner_get_backup_set_list(const char *cluster_name, const int64_t cluster_id,
     const int64_t restore_timestamp, const ObBackupDest &backup_dest, const ObArray<ObBackupSetFileInfo> &file_infos,
-    common::ObArray<ObSimpleBackupSetPath> &path_list, int64_t &snapshot_version, int64_t &start_replay_log_ts)
+    common::ObArray<ObSimpleBackupSetPath> &path_list, int64_t &snapshot_version, int64_t &start_replay_log_ts,
+    bool &is_snapshot_restore)
 {
   int ret = OB_SUCCESS;
   snapshot_version = -1;
@@ -939,6 +953,8 @@ int ObMultiBackupDestUtil::do_inner_get_backup_set_list(const char *cluster_name
     const ObBackupSetFileInfo &info = file_infos.at(i);
     if (OB_SUCCESS != info.result_) {
       // do nothing
+    } else if (OB_FAIL(check_is_snapshot_restore(info, restore_timestamp, is_snapshot_restore))) {
+      LOG_WARN("failed to check is snaptshot backup", K(ret), K(info), K(restore_timestamp));
     } else if (!ObBackupFileStatus::can_show_in_preview(info.file_status_)) {
       LOG_INFO("backup set info cannot list in preview", K(info));
     } else {
@@ -1426,5 +1442,18 @@ int ObMultiBackupDestUtil::check_backup_path_is_backup_backup(const char *cluste
   return ret;
 }
 
+int ObMultiBackupDestUtil::check_is_snapshot_restore(
+    const share::ObBackupSetFileInfo &backup_set_info, const int64_t restore_timestamp, bool &is_snapshot_restore)
+{
+  int ret = OB_SUCCESS;
+  if (!backup_set_info.is_valid() || restore_timestamp <= 0) {
+    ret = OB_INVALID_ARGUMENT;
+    LOG_WARN("get invalid args", K(ret), K(backup_set_info), K(restore_timestamp));
+  } else {
+    is_snapshot_restore = backup_set_info.snapshot_version_ == restore_timestamp;
+  }
+  return ret;
+}
+
 }  // end namespace share
 }  // end namespace oceanbase

src/share/backup/ob_multi_backup_dest_util.h

@@ -89,18 +89,19 @@ private:
   static int get_backup_set_list(const bool is_preview, const char *cluster_name, const int64_t cluster_id,
       const uint64_t tenant_id, const int64_t restore_timestamp, const common::ObString &backup_dest_str,
       common::ObArray<ObSimpleBackupSetPath> &path_list, int64_t &snapshot_version, int64_t &start_replay_log_ts,
-      bool &is_compat_path);
+      bool &is_compat_path, bool &is_snapshot_restore);
   static int do_get_backup_set_list(const bool is_preview, const char *cluster_name, const int64_t cluster_id,
       const uint64_t tenant_id, const int64_t restore_timestamp, const ObBackupDest &backup_dest,
       common::ObArray<ObSimpleBackupSetPath> &path_list, int64_t &snapshot_version, int64_t &start_replay_log_ts,
-      bool &is_compat_path);
+      bool &is_compat_path, bool &is_snapshot_restore);
   static int do_get_backup_set_list_from_cluster_level(const bool is_preview, const char *cluster_name,
       const int64_t cluster_id, const uint64_t tenant_id, const int64_t restore_timestamp,
       const ObBackupDest &backup_dest, common::ObArray<ObSimpleBackupSetPath> &path_list, int64_t &snapshot_version,
-      int64_t &start_replay_log_ts);
+      int64_t &start_replay_log_ts, bool &is_snapshot_restore);
   static int do_inner_get_backup_set_list(const char *cluster_name, const int64_t cluster_id,
       const int64_t restore_timestamp, const ObBackupDest &backup_dest, const ObArray<ObBackupSetFileInfo> &file_infos,
-      common::ObArray<ObSimpleBackupSetPath> &path_list, int64_t &snapshot_version, int64_t &start_replay_log_ts);
+      common::ObArray<ObSimpleBackupSetPath> &path_list, int64_t &snapshot_version, int64_t &start_replay_log_ts,
+      bool &is_snapshot_restore);
   static int get_backup_piece_list(const bool is_preview, const char *cluster_name, const int64_t cluster_id,
       const uint64_t tenant_id, const int64_t snapshot_version, const int64_t start_replay_log_ts,
       const int64_t restore_timestamp, const common::ObString &backup_dest_str,
@@ -125,6 +126,8 @@ private:
   static int check_backup_path_is_backup_backup(const char *cluster_name, const int64_t cluster_id,
       const common::ObString &root_path, const common::ObString &storage_info, const uint64_t tenant_id,
       bool &is_backup_backup);
+  static int check_is_snapshot_restore(
+      const share::ObBackupSetFileInfo &backup_set_info, const int64_t restore_timestamp, bool &is_snapshot_restore);
 
 private:
   DISALLOW_COPY_AND_ASSIGN(ObMultiBackupDestUtil);