database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[improvement](balance) fix multiple problems for balance on large cluster (#30713)

Browse Source
This commit is contained in:
yujun
2024-02-07 07:02:41 +08:00
committed by yiguolei
parent 5295e16727
commit 4052746f1c
11 changed files with 544 additions and 134 deletions
Download Patch File Download Diff File Expand all files Collapse all files

31
fe/fe-common/src/main/java/org/apache/doris/common/Config.java
View File

@ -1014,6 +1014,30 @@ public class Config extends ConfigBase {
@ConfField(mutable = true, masterOnly = true)
public static double balance_load_score_threshold = 0.1; // 10%
// if disk usage > balance_load_score_threshold + urgent_disk_usage_extra_threshold
// then this disk need schedule quickly
// this value could less than 0.
@ConfField(mutable = true, masterOnly = true)
public static double urgent_balance_disk_usage_extra_threshold = 0.05;
// when run urgent disk balance, shuffle the top large tablets
// range: [ 0 ~ 100 ]
@ConfField(mutable = true, masterOnly = true)
public static int urgent_balance_shuffle_large_tablet_percentage = 1;
@ConfField(mutable = true, masterOnly = true)
public static double urgent_balance_pick_large_tablet_num_threshold = 1000;
// range: 0 ~ 100
@ConfField(mutable = true, masterOnly = true)
public static int urgent_balance_pick_large_disk_usage_percentage = 80;
// there's a case, all backend has a high disk, by default, it will not run urgent disk balance.
// if set this value to true, urgent disk balance will always run,
// the backends will exchange tablets among themselves.
@ConfField(mutable = true, masterOnly = true)
public static boolean enable_urgent_balance_no_low_backend = true;
/**
* if set to true, TabletScheduler will not do balance.
*/
@ -1024,7 +1048,7 @@ public class Config extends ConfigBase {
* when be rebalancer idle, then disk balance will occurs.
*/
@ConfField(mutable = true, masterOnly = true)
public static int be_rebalancer_idle_seconds = 60;
public static int be_rebalancer_idle_seconds = 0;
/**
* if set to true, TabletScheduler will not do disk balance.
@ -1032,6 +1056,11 @@ public class Config extends ConfigBase {
@ConfField(mutable = true, masterOnly = true)
public static boolean disable_disk_balance = false;
// balance order
// ATTN: a temporary config, may delete later.
@ConfField(mutable = true, masterOnly = true)
public static boolean balance_be_then_disk = true;
/**
* if set to false, TabletScheduler will not do disk balance for replica num = 1.
*/

2
fe/fe-common/src/main/java/org/apache/doris/common/Pair.java
View File

@ -78,7 +78,7 @@ public class Pair<F, S> {
return first.toString() + ":" + second.toString();
}
public static class PairComparator<T extends Pair<?, Comparable>> implements Comparator<T> {
public static class PairComparator<T extends Pair<?, ? extends Comparable>> implements Comparator<T> {
@Override
public int compare(T o1, T o2) {
return o1.second.compareTo(o2.second);