diff --git a/docs/en/docs/admin-manual/config/fe-config.md b/docs/en/docs/admin-manual/config/fe-config.md index e88e8f840b..2fd5b58d9f 100644 --- a/docs/en/docs/admin-manual/config/fe-config.md +++ b/docs/en/docs/admin-manual/config/fe-config.md @@ -2557,6 +2557,26 @@ MasterOnly:true default timeout of backup job +#### `backup_upload_task_num_per_be` + +Default:3 + +IsMutable:true + +MasterOnly:true + +The max number of upload tasks assigned to each be during the backup process, the default value is 3. + +#### `restore_download_task_num_per_be` + +Default:3 + +IsMutable:true + +MasterOnly:true + +The max number of download tasks assigned to each be during the restore process, the default value is 3. + #### `max_backup_restore_job_num_per_db` Default: 10 diff --git a/docs/zh-CN/docs/admin-manual/config/fe-config.md b/docs/zh-CN/docs/admin-manual/config/fe-config.md index 6b8c9f8a85..47bcc9fba9 100644 --- a/docs/zh-CN/docs/admin-manual/config/fe-config.md +++ b/docs/zh-CN/docs/admin-manual/config/fe-config.md @@ -2558,6 +2558,26 @@ SmallFileMgr 中存储的最大文件数 备份作业的默认超时时间 +#### `backup_upload_task_num_per_be` + +默认值:3 + +是否可以动态配置:true + +是否为 Master FE 节点独有的配置项:true + +备份过程中,分配给每个be的upload任务最大个数,默认值为3个。 + +#### `restore_download_task_num_per_be` + +默认值:3 + +是否可以动态配置:true + +是否为 Master FE 节点独有的配置项:true + +恢复过程中,分配给每个be的download任务最大个数,默认值为3个。 + #### `max_backup_restore_job_num_per_db` 默认值:10 diff --git a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java index 231d652559..d6be0fb919 100644 --- a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java +++ b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java @@ -2298,6 +2298,18 @@ public class Config extends ConfigBase { "The path of the nereids trace file."}) public static String nereids_trace_log_dir = System.getenv("DORIS_HOME") + "/log/nereids_trace"; + @ConfField(mutable = true, masterOnly = true, description = { + "备份过程中,分配给每个be的upload任务最大个数,默认值为3个。", + "The max number of upload tasks assigned to each be during the backup process, the default value is 3." + }) + public static int backup_upload_task_num_per_be = 3; + + @ConfField(mutable = true, masterOnly = true, description = { + "恢复过程中,分配给每个be的download任务最大个数,默认值为3个。", + "The max number of download tasks assigned to each be during the restore process, the default value is 3." + }) + public static int restore_download_task_num_per_be = 3; + @ConfField(description = {"是否开启通过http接口获取log文件的功能", "Whether to enable the function of getting log files through http interface"}) public static boolean enable_get_log_file_api = false; diff --git a/fe/fe-core/src/main/java/org/apache/doris/backup/BackupJob.java b/fe/fe-core/src/main/java/org/apache/doris/backup/BackupJob.java index a91c2e12a5..d764b926e7 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/backup/BackupJob.java +++ b/fe/fe-core/src/main/java/org/apache/doris/backup/BackupJob.java @@ -34,6 +34,7 @@ import org.apache.doris.catalog.Resource; import org.apache.doris.catalog.Table; import org.apache.doris.catalog.Tablet; import org.apache.doris.catalog.View; +import org.apache.doris.common.Config; import org.apache.doris.common.io.Text; import org.apache.doris.common.util.TimeUtils; import org.apache.doris.datasource.property.S3ClientBEProperties; @@ -614,8 +615,7 @@ public class BackupJob extends AbstractJob { for (Long beId : beToSnapshots.keySet()) { List infos = beToSnapshots.get(beId); int totalNum = infos.size(); - // each backend allot at most 3 tasks - int batchNum = Math.min(totalNum, 3); + int batchNum = Math.min(totalNum, Config.backup_upload_task_num_per_be); // each task contains several upload sub tasks int taskNumPerBatch = Math.max(totalNum / batchNum, 1); LOG.info("backend {} has {} batch, total {} tasks, {}", beId, batchNum, totalNum, this); diff --git a/fe/fe-core/src/main/java/org/apache/doris/backup/RestoreJob.java b/fe/fe-core/src/main/java/org/apache/doris/backup/RestoreJob.java index a69189ba10..df5d96dead 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/backup/RestoreJob.java +++ b/fe/fe-core/src/main/java/org/apache/doris/backup/RestoreJob.java @@ -52,6 +52,7 @@ import org.apache.doris.catalog.Tablet; import org.apache.doris.catalog.TabletMeta; import org.apache.doris.catalog.View; import org.apache.doris.clone.DynamicPartitionScheduler; +import org.apache.doris.common.Config; import org.apache.doris.common.DdlException; import org.apache.doris.common.FeMetaVersion; import org.apache.doris.common.MarkedCountDownLatch; @@ -1337,8 +1338,7 @@ public class RestoreJob extends AbstractJob { for (Long beId : beToSnapshots.keySet()) { List beSnapshotInfos = beToSnapshots.get(beId); int totalNum = beSnapshotInfos.size(); - // each backend allot at most 3 tasks - int batchNum = Math.min(totalNum, 3); + int batchNum = Math.min(totalNum, Config.restore_download_task_num_per_be); // each task contains several upload sub tasks int taskNumPerBatch = Math.max(totalNum / batchNum, 1); LOG.debug("backend {} has {} batch, total {} tasks, {}", @@ -1490,8 +1490,7 @@ public class RestoreJob extends AbstractJob { for (Long beId : beToSnapshots.keySet()) { List beSnapshotInfos = beToSnapshots.get(beId); int totalNum = beSnapshotInfos.size(); - // each backend allot at most 3 tasks - int batchNum = Math.min(totalNum, 3); + int batchNum = Math.min(totalNum, Config.restore_download_task_num_per_be); // each task contains several upload sub tasks int taskNumPerBatch = Math.max(totalNum / batchNum, 1);