diff --git a/fe/fe-core/src/main/java/org/apache/doris/backup/BackupJobInfo.java b/fe/fe-core/src/main/java/org/apache/doris/backup/BackupJobInfo.java
index c24d6c0329..7108315cee 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/backup/BackupJobInfo.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/backup/BackupJobInfo.java
@@ -788,6 +788,20 @@ public class BackupJobInfo implements Writable {
         return getBrief();
     }
 
+    public void releaseSnapshotInfo() {
+        tabletBeMap.clear();
+        tabletSnapshotPathMap.clear();
+        for (BackupOlapTableInfo tableInfo : backupOlapTableObjects.values()) {
+            for (BackupPartitionInfo partInfo : tableInfo.partitions.values()) {
+                for (BackupIndexInfo indexInfo : partInfo.indexes.values()) {
+                    for (BackupTabletInfo tabletInfo : indexInfo.sortedTabletInfoList) {
+                        tabletInfo.files.clear();
+                    }
+                }
+            }
+        }
+    }
+
     public static BackupJobInfo read(DataInput in) throws IOException {
         return BackupJobInfo.readFields(in);
     }
diff --git a/fe/fe-core/src/main/java/org/apache/doris/backup/RestoreFileMapping.java b/fe/fe-core/src/main/java/org/apache/doris/backup/RestoreFileMapping.java
index bcc3b7477b..07ddf6844d 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/backup/RestoreFileMapping.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/backup/RestoreFileMapping.java
@@ -156,6 +156,11 @@ public class RestoreFileMapping implements Writable {
         return mapping;
     }
 
+    public void clear() {
+        mapping.clear();
+        overwriteMap.clear();
+    }
+
     @Override
     public void write(DataOutput out) throws IOException {
         out.writeInt(mapping.size());
diff --git a/fe/fe-core/src/main/java/org/apache/doris/backup/RestoreJob.java b/fe/fe-core/src/main/java/org/apache/doris/backup/RestoreJob.java
index 9dce692d17..5eb4950466 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/backup/RestoreJob.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/backup/RestoreJob.java
@@ -1801,6 +1801,8 @@ public class RestoreJob extends AbstractJob {
             releaseSnapshots();
 
             snapshotInfos.clear();
+            fileMapping.clear();
+            jobInfo.releaseSnapshotInfo();
 
             finishedTime = System.currentTimeMillis();
             state = RestoreJobState.FINISHED;
@@ -1993,6 +1995,9 @@ public class RestoreJob extends AbstractJob {
             releaseSnapshots();
 
             snapshotInfos.clear();
+            fileMapping.clear();
+            jobInfo.releaseSnapshotInfo();
+
             RestoreJobState curState = state;
             finishedTime = System.currentTimeMillis();
             state = RestoreJobState.CANCELLED;