database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[enhancement](backup-restore) add config for upload/download task num per be (#27772)

Browse Source 
set upload/download task num per be, and improve the overall speed of upload/download, enhance the performance of backup and recovery.

---------

Signed-off-by: nextdreamblue <zxw520blue1@163.com>
This commit is contained in:
xueweizhang
2023-12-04 11:19:45 +08:00
committed by GitHub
parent e3d2425d47
commit 80f528bf26
5 changed files with 57 additions and 6 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
fe/fe-core/src/main/java/org/apache/doris/backup/BackupJob.java
View File

@ -34,6 +34,7 @@ import org.apache.doris.catalog.Resource;
import org.apache.doris.catalog.Table;
import org.apache.doris.catalog.Tablet;
import org.apache.doris.catalog.View;
import org.apache.doris.common.Config;
import org.apache.doris.common.io.Text;
import org.apache.doris.common.util.TimeUtils;
import org.apache.doris.datasource.property.S3ClientBEProperties;
@ -614,8 +615,7 @@ public class BackupJob extends AbstractJob {
for (Long beId : beToSnapshots.keySet()) {
List<SnapshotInfo> infos = beToSnapshots.get(beId);
int totalNum = infos.size();
// each backend allot at most 3 tasks
int batchNum = Math.min(totalNum, 3);
int batchNum = Math.min(totalNum, Config.backup_upload_task_num_per_be);
// each task contains several upload sub tasks
int taskNumPerBatch = Math.max(totalNum / batchNum, 1);
LOG.info("backend {} has {} batch, total {} tasks, {}", beId, batchNum, totalNum, this);

7
fe/fe-core/src/main/java/org/apache/doris/backup/RestoreJob.java
View File

@ -52,6 +52,7 @@ import org.apache.doris.catalog.Tablet;
import org.apache.doris.catalog.TabletMeta;
import org.apache.doris.catalog.View;
import org.apache.doris.clone.DynamicPartitionScheduler;
import org.apache.doris.common.Config;
import org.apache.doris.common.DdlException;
import org.apache.doris.common.FeMetaVersion;
import org.apache.doris.common.MarkedCountDownLatch;
@ -1337,8 +1338,7 @@ public class RestoreJob extends AbstractJob {
for (Long beId : beToSnapshots.keySet()) {
List<SnapshotInfo> beSnapshotInfos = beToSnapshots.get(beId);
int totalNum = beSnapshotInfos.size();
// each backend allot at most 3 tasks
int batchNum = Math.min(totalNum, 3);
int batchNum = Math.min(totalNum, Config.restore_download_task_num_per_be);
// each task contains several upload sub tasks
int taskNumPerBatch = Math.max(totalNum / batchNum, 1);
LOG.debug("backend {} has {} batch, total {} tasks, {}",
@ -1490,8 +1490,7 @@ public class RestoreJob extends AbstractJob {
for (Long beId : beToSnapshots.keySet()) {
List<SnapshotInfo> beSnapshotInfos = beToSnapshots.get(beId);
int totalNum = beSnapshotInfos.size();
// each backend allot at most 3 tasks
int batchNum = Math.min(totalNum, 3);
int batchNum = Math.min(totalNum, Config.restore_download_task_num_per_be);
// each task contains several upload sub tasks
int taskNumPerBatch = Math.max(totalNum / batchNum, 1);