database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

Make MAX_SCHEDULING_TABLETS and MAX_BALANCING_TABLETS configurable (#2670)

Browse Source
This commit is contained in:
kangkaisen
2020-01-06 20:15:38 +08:00
committed by ZHAO Chun
parent 4d6afdae4d
commit 9ca7fdfe1c
3 changed files with 18 additions and 13 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
fe/src/main/java/org/apache/doris/clone/TabletChecker.java
View File

@ -31,6 +31,7 @@ import org.apache.doris.catalog.Table.TableType;
import org.apache.doris.catalog.Tablet;
import org.apache.doris.catalog.Tablet.TabletStatus;
import org.apache.doris.clone.TabletScheduler.AddResult;
import org.apache.doris.common.Config;
import org.apache.doris.common.DdlException;
import org.apache.doris.common.Pair;
import org.apache.doris.common.util.MasterDaemon;
@ -157,10 +158,10 @@ public class TabletChecker extends MasterDaemon {
protected void runAfterCatalogReady() {
int pendingNum = tabletScheduler.getPendingNum();
int runningNum = tabletScheduler.getRunningNum();
if (pendingNum > TabletScheduler.MAX_SCHEDULING_TABLETS
|| runningNum > TabletScheduler.MAX_SCHEDULING_TABLETS) {
if (pendingNum > Config.max_scheduling_tablets
|| runningNum > Config.max_scheduling_tablets) {
LOG.info("too many tablets are being scheduled. pending: {}, running: {}, limit: {}. skip check",
pendingNum, runningNum, TabletScheduler.MAX_SCHEDULING_TABLETS);
pendingNum, runningNum, Config.max_scheduling_tablets);
return;
}

14
fe/src/main/java/org/apache/doris/clone/TabletScheduler.java
View File

@ -96,13 +96,6 @@ public class TabletScheduler extends MasterDaemon {
public static final int BALANCE_SLOT_NUM_FOR_PATH = 2;
// if the number of scheduled tablets in TabletScheduler exceed this threshold,
// skip checking.
public static final int MAX_SCHEDULING_TABLETS = 2000;
// if the number of balancing tablets in TabletScheduler exceed this threshold,
// no more balance check
public static final int MAX_BALANCING_TABLETS = 100;
/*
* Tablet is added to pendingTablets as well it's id in allTabletIds.
* TabletScheduler will take tablet from pendingTablets but will not remove it's id from allTabletIds when
@ -220,7 +213,8 @@ public class TabletScheduler extends MasterDaemon {
// and number of scheduling tablets exceed the limit,
// refuse to add.
if (tablet.getType() != TabletSchedCtx.Type.BALANCE && !force
&& (pendingTablets.size() > MAX_SCHEDULING_TABLETS || runningTablets.size() > MAX_SCHEDULING_TABLETS)) {
&& (pendingTablets.size() > Config.max_scheduling_tablets
|| runningTablets.size() > Config.max_scheduling_tablets)) {
return AddResult.LIMIT_EXCEED;
}
@ -976,9 +970,9 @@ public class TabletScheduler extends MasterDaemon {
}
long numOfBalancingTablets = getBalanceTabletsNumber();
if (numOfBalancingTablets > MAX_BALANCING_TABLETS) {
if (numOfBalancingTablets > Config.max_balancing_tablets) {
LOG.info("number of balancing tablets {} exceed limit: {}, skip selecting tablets for balance",
numOfBalancingTablets, MAX_BALANCING_TABLETS);
numOfBalancingTablets, Config.max_balancing_tablets);
return;
}

10
fe/src/main/java/org/apache/doris/common/Config.java
View File

@ -846,6 +846,16 @@ public class Config extends ConfigBase {
*/
@ConfField(mutable = true, masterOnly = true)
public static boolean disable_balance = false;
// if the number of scheduled tablets in TabletScheduler exceed max_scheduling_tablets
// skip checking.
@ConfField(mutable = true, masterOnly = true)
public static int max_scheduling_tablets = 2000;
// if the number of balancing tablets in TabletScheduler exceed max_balancing_tablets,
// no more balance check
@ConfField(mutable = true, masterOnly = true)
public static int max_balancing_tablets = 100;
// This threshold is to avoid piling up too many report task in FE, which may cause OOM exception.
// In some large Doris cluster, eg: 100 Backends with ten million replicas, a tablet report may cost