database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[fix](restore) fix restore odbc resource bug (#31989)

Browse Source 
if some odbc tables use the same resource, and restore not all odbc tables, it will not retain the resource.
and check some conf for backup/restore

Signed-off-by: nextdreamblue <zxw520blue1@163.com>
This commit is contained in:
xueweizhang
2024-03-22 10:34:54 +08:00
committed by yiguolei
parent 4de8775e17
commit b2db5a4ded
3 changed files with 16 additions and 6 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
fe/fe-core/src/main/java/org/apache/doris/backup/BackupJob.java
View File

@ -643,7 +643,10 @@ public class BackupJob extends AbstractJob {
for (Long beId : beToSnapshots.keySet()) {
List<SnapshotInfo> infos = beToSnapshots.get(beId);
int totalNum = infos.size();
int batchNum = Math.min(totalNum, Config.backup_upload_task_num_per_be);
int batchNum = totalNum;
if (Config.backup_upload_task_num_per_be > 0) {
batchNum = Math.min(totalNum, Config.backup_upload_task_num_per_be);
}
// each task contains several upload sub tasks
int taskNumPerBatch = Math.max(totalNum / batchNum, 1);
LOG.info("backend {} has {} batch, total {} tasks, {}", beId, batchNum, totalNum, this);

7
fe/fe-core/src/main/java/org/apache/doris/backup/BackupJobInfo.java
View File

@ -277,17 +277,18 @@ public class BackupJobInfo implements Writable {
public void retainOdbcTables(Set<String> odbcTableNames) {
Iterator<BackupOdbcTableInfo> odbcIter = newBackupObjects.odbcTables.listIterator();
Set<String> removedOdbcResourceNames = Sets.newHashSet();
Set<String> retainOdbcResourceNames = Sets.newHashSet();
while (odbcIter.hasNext()) {
BackupOdbcTableInfo backupOdbcTableInfo = odbcIter.next();
if (!odbcTableNames.contains(backupOdbcTableInfo.dorisTableName)) {
removedOdbcResourceNames.add(backupOdbcTableInfo.resourceName);
odbcIter.remove();
} else {
retainOdbcResourceNames.add(backupOdbcTableInfo.resourceName);
}
}
Iterator<BackupOdbcResourceInfo> resourceIter = newBackupObjects.odbcResources.listIterator();
while (resourceIter.hasNext()) {
if (removedOdbcResourceNames.contains(resourceIter.next().name)) {
if (!retainOdbcResourceNames.contains(resourceIter.next().name)) {
resourceIter.remove();
}
}

10
fe/fe-core/src/main/java/org/apache/doris/backup/RestoreJob.java
View File

@ -1381,7 +1381,10 @@ public class RestoreJob extends AbstractJob {
for (Long beId : beToSnapshots.keySet()) {
List<SnapshotInfo> beSnapshotInfos = beToSnapshots.get(beId);
int totalNum = beSnapshotInfos.size();
int batchNum = Math.min(totalNum, Config.restore_download_task_num_per_be);
int batchNum = totalNum;
if (Config.restore_download_task_num_per_be > 0) {
batchNum = Math.min(totalNum, Config.restore_download_task_num_per_be);
}
// each task contains several upload sub tasks
int taskNumPerBatch = Math.max(totalNum / batchNum, 1);
if (LOG.isDebugEnabled()) {
@ -1537,7 +1540,10 @@ public class RestoreJob extends AbstractJob {
for (Long beId : beToSnapshots.keySet()) {
List<SnapshotInfo> beSnapshotInfos = beToSnapshots.get(beId);
int totalNum = beSnapshotInfos.size();
int batchNum = Math.min(totalNum, Config.restore_download_task_num_per_be);
int batchNum = totalNum;
if (Config.restore_download_task_num_per_be > 0) {
batchNum = Math.min(totalNum, Config.restore_download_task_num_per_be);
}
// each task contains several upload sub tasks
int taskNumPerBatch = Math.max(totalNum / batchNum, 1);