database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[fix](tablet clone) clone add replica prefer choose the same medium (#25640)

Browse Source
This commit is contained in:
yujun
2023-10-24 15:18:32 +08:00
committed by GitHub
parent c0f8c0af39
commit a2d82220e2
9 changed files with 133 additions and 36 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
fe/fe-core/src/main/java/org/apache/doris/clone/BackendLoadStatistic.java
View File

@ -414,7 +414,7 @@ public class BackendLoadStatistic {
for (int i = 0; i < pathStatistics.size(); i++) {
RootPathLoadStatistic pathStatistic = pathStatistics.get(i);
// if this is a supplement task, ignore the storage medium
if (!isSupplement && pathStatistic.getStorageMedium() != medium) {
if (!isSupplement && medium != null && pathStatistic.getStorageMedium() != medium) {
LOG.debug("backend {} path {}'s storage medium {} is not {} storage medium, actual: {}",
beId, pathStatistic.getPath(), pathStatistic.getStorageMedium(), medium);
continue;

2
fe/fe-core/src/main/java/org/apache/doris/clone/ColocateTableCheckerAndBalancer.java
View File

@ -866,7 +866,7 @@ public class ColocateTableCheckerAndBalancer extends MasterDaemon {
globalColocateStatistic.getBucketTotalReplicaDataSize(groupId, bucketIndex);
resultPaths.clear();
BalanceStatus st = beStat.isFit(bucketDataSize, null, resultPaths, true);
BalanceStatus st = beStat.isFit(bucketDataSize, null, resultPaths, false);
if (!st.ok()) {
LOG.debug("backend {} is unable to fit in group {}, tablet order idx {}, data size {}",
destBeId, groupId, bucketIndex, bucketDataSize);

10
fe/fe-core/src/main/java/org/apache/doris/clone/RootPathLoadStatistic.java
View File

@ -98,9 +98,11 @@ public class RootPathLoadStatistic implements Comparable<RootPathLoadStatistic>
toString() + " does not fit tablet with size: " + tabletSize + ", offline");
}
double newUsagePerc = (usedCapacityB + tabletSize) / (double) capacityB;
long newLeftCapacity = capacityB - usedCapacityB - tabletSize;
if (isSupplement) {
if ((usedCapacityB + tabletSize) / (double) capacityB > (Config.storage_flood_stage_usage_percent / 100.0)
&& capacityB - usedCapacityB - tabletSize < Config.storage_flood_stage_left_capacity_bytes) {
if (newUsagePerc > (Config.storage_flood_stage_usage_percent / 100.0)
|| newLeftCapacity < Config.storage_flood_stage_left_capacity_bytes) {
return new BalanceStatus(ErrCode.COMMON_ERROR,
toString() + " does not fit tablet with size: " + tabletSize + ", limitation reached");
} else {
@ -108,8 +110,8 @@ public class RootPathLoadStatistic implements Comparable<RootPathLoadStatistic>
}
}
if ((usedCapacityB + tabletSize) / (double) capacityB > (Config.storage_high_watermark_usage_percent / 100.0)
|| capacityB - usedCapacityB - tabletSize < Config.storage_min_left_capacity_bytes) {
if (newUsagePerc > (Config.storage_high_watermark_usage_percent / 100.0)
|| newLeftCapacity < Config.storage_min_left_capacity_bytes) {
return new BalanceStatus(ErrCode.COMMON_ERROR,
toString() + " does not fit tablet with size: " + tabletSize);
}

42
fe/fe-core/src/main/java/org/apache/doris/clone/TabletScheduler.java
View File

@ -1318,7 +1318,8 @@ public class TabletScheduler extends MasterDaemon {
// get all available paths which this tablet can fit in.
// beStatistics is sorted by mix load score in ascend order, so select from first to last.
List<BePathLoadStatPair> allFitPaths = Lists.newArrayList();
List<BePathLoadStatPair> allFitPathsSameMedium = Lists.newArrayList();
List<BePathLoadStatPair> allFitPathsDiffMedium = Lists.newArrayList();
for (BackendLoadStatistic bes : beStatistics) {
if (!bes.isAvailable()) {
LOG.debug("backend {} is not available, skip. tablet: {}", bes.getBeId(), tabletCtx.getTabletId());
@ -1349,27 +1350,27 @@ public class TabletScheduler extends MasterDaemon {
List<RootPathLoadStatistic> resultPaths = Lists.newArrayList();
BalanceStatus st = bes.isFit(tabletCtx.getTabletSize(), tabletCtx.getStorageMedium(),
resultPaths, tabletCtx.getTabletStatus() != TabletStatus.REPLICA_RELOCATING
/* if REPLICA_RELOCATING, then it is not a supplement task */);
if (!st.ok()) {
LOG.debug("unable to find path for tablet: {}. {}", tabletCtx, st);
// This is to solve, when we decommission some BEs with SSD disks,
// if there are no SSD disks on the remaining BEs, it will be impossible to select a
// suitable destination path.
// In this case, we need to ignore the storage medium property
// and try to select the destination path again.
// Set `isSupplement` to true will ignore the storage medium property.
st = bes.isFit(tabletCtx.getTabletSize(), tabletCtx.getStorageMedium(),
resultPaths, true);
if (!st.ok()) {
LOG.debug("unable to find path for supplementing tablet: {}. {}", tabletCtx, st);
continue;
resultPaths, false);
if (st.ok()) {
resultPaths.stream().forEach(path -> allFitPathsSameMedium.add(new BePathLoadStatPair(bes, path)));
} else {
LOG.debug("backend {} unable to find path for tablet: {}. {}", bes.getBeId(), tabletCtx, st);
resultPaths.clear();
st = bes.isFit(tabletCtx.getTabletSize(), tabletCtx.getStorageMedium(), resultPaths, true);
if (st.ok()) {
resultPaths.stream().forEach(path -> allFitPathsDiffMedium.add(new BePathLoadStatPair(bes, path)));
} else {
LOG.debug("backend {} unable to find path for supplementing tablet: {}. {}",
bes.getBeId(), tabletCtx, st);
}
}
resultPaths.stream().forEach(path -> allFitPaths.add(new BePathLoadStatPair(bes, path)));
}
// all fit paths has already been sorted by load score in 'allFitPaths' in ascend order.
// just get first available path.
// we try to find a path with specified media type, if not find, arbitrarily use one.
List<BePathLoadStatPair> allFitPaths =
!allFitPathsSameMedium.isEmpty() ? allFitPathsSameMedium : allFitPathsDiffMedium;
if (allFitPaths.isEmpty()) {
throw new SchedException(Status.UNRECOVERABLE, "unable to find dest path for new replica");
}
@ -1377,14 +1378,11 @@ public class TabletScheduler extends MasterDaemon {
BePathLoadStatPairComparator comparator = new BePathLoadStatPairComparator(allFitPaths);
Collections.sort(allFitPaths, comparator);
// all fit paths has already been sorted by load score in 'allFitPaths' in ascend order.
// just get first available path.
// we try to find a path with specified media type, if not find, arbitrarily use one.
for (BePathLoadStatPair bePathLoadStat : allFitPaths) {
RootPathLoadStatistic rootPathLoadStatistic = bePathLoadStat.getPathLoadStatistic();
if (rootPathLoadStatistic.getStorageMedium() != tabletCtx.getStorageMedium()) {
LOG.debug("backend {}'s path {}'s storage medium {} "
+ "is not equal to tablet's storage medium {}, skip. tablet: {}",
+ "is not equal to tablet's storage medium {}, skip. tablet: {}",
rootPathLoadStatistic.getBeId(), rootPathLoadStatistic.getPathHash(),
rootPathLoadStatistic.getStorageMedium(), tabletCtx.getStorageMedium(),
tabletCtx.getTabletId());