database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[fix] Fix disk used pct only consider the data that used by Doris (#9705)

Browse Source
This commit is contained in:
ccoffline
2022-06-15 16:28:56 +08:00
committed by GitHub
parent f1d0c231b9
commit 49f4437396
4 changed files with 21 additions and 9 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
fe/fe-core/src/main/java/org/apache/doris/catalog/DiskInfo.java
View File

@ -89,6 +89,10 @@ public class DiskInfo implements Writable {
this.dataUsedCapacityB = dataUsedCapacityB;
}
public long getDiskUsedCapacityB() {
return totalCapacityB - diskAvailableCapacityB;
}
public long getAvailableCapacityB() {
return diskAvailableCapacityB;
}
@ -98,7 +102,7 @@ public class DiskInfo implements Writable {
}
public double getUsedPct() {
return (totalCapacityB - diskAvailableCapacityB) / (double) (totalCapacityB <= 0 ? 1 : totalCapacityB);
return this.getDiskUsedCapacityB() / (double) (totalCapacityB <= 0 ? 1 : totalCapacityB);
}
public DiskState getState() {
@ -148,12 +152,10 @@ public class DiskInfo implements Writable {
floodStage, diskAvailableCapacityB, totalCapacityB);
if (floodStage) {
return diskAvailableCapacityB < Config.storage_flood_stage_left_capacity_bytes
&& (double) (totalCapacityB - diskAvailableCapacityB) / totalCapacityB
> (Config.storage_flood_stage_usage_percent / 100.0);
&& this.getUsedPct() > (Config.storage_flood_stage_usage_percent / 100.0);
} else {
return diskAvailableCapacityB < Config.storage_min_left_capacity_bytes
|| (double) (totalCapacityB - diskAvailableCapacityB) / totalCapacityB
> (Config.storage_high_watermark_usage_percent / 100.0);
|| this.getUsedPct() > (Config.storage_high_watermark_usage_percent / 100.0);
}
}

2
fe/fe-core/src/main/java/org/apache/doris/clone/BackendLoadStatistic.java
View File

@ -187,7 +187,7 @@ public class BackendLoadStatistic {
RootPathLoadStatistic pathStatistic = new RootPathLoadStatistic(beId, diskInfo.getRootPath(),
diskInfo.getPathHash(), diskInfo.getStorageMedium(),
diskInfo.getTotalCapacityB(), diskInfo.getDataUsedCapacityB(), diskInfo.getState());
diskInfo.getTotalCapacityB(), diskInfo.getDiskUsedCapacityB(), diskInfo.getState());
pathStatistics.add(pathStatistic);
}

15
fe/fe-core/src/test/java/org/apache/doris/clone/DiskRebalanceTest.java
View File

@ -190,9 +190,12 @@ public class DiskRebalanceTest {
@Test
public void testDiskRebalancerWithDiffUsageDisk() {
// init system
systemInfoService.addBackend(RebalancerTestUtil.createBackend(10001L, 2048, Lists.newArrayList(1024L), 1));
systemInfoService.addBackend(RebalancerTestUtil.createBackend(10002L, 2048, Lists.newArrayList(1024L, 512L), 2));
systemInfoService.addBackend(RebalancerTestUtil.createBackend(10003L, 2048, Lists.newArrayList(1024L, 512L, 513L), 3));
systemInfoService.addBackend(RebalancerTestUtil.createBackend(10001L, 2048,
Lists.newArrayList(1024L), 1));
systemInfoService.addBackend(RebalancerTestUtil.createBackend(10002L, 2048,
Lists.newArrayList(1024L, 512L), 2));
systemInfoService.addBackend(RebalancerTestUtil.createBackend(10003L, 2048,
Lists.newArrayList(1024L, 512L, 1024L), 3));
olapTable = new OlapTable(2, "fake table", new ArrayList<>(), KeysType.DUP_KEYS,
new RangePartitionInfo(), new HashDistributionInfo());
@ -221,6 +224,12 @@ public class DiskRebalanceTest {
Rebalancer rebalancer = new DiskRebalancer(Catalog.getCurrentSystemInfo(), Catalog.getCurrentInvertedIndex());
generateStatisticMap();
rebalancer.updateLoadStatistic(statisticMap);
for (Table.Cell<String, Tag, ClusterLoadStatistic> s : statisticMap.cellSet()) {
if (s.getValue() != null) {
LOG.info("cluster = {}, tag = {}, statistic = {}",
s.getRowKey(), s.getColumnKey(), s.getValue().getBrief());
}
}
List<TabletSchedCtx> alternativeTablets = rebalancer.selectAlternativeTablets();
// check alternativeTablets;
Assert.assertEquals(2, alternativeTablets.size());

1
fe/fe-core/src/test/java/org/apache/doris/clone/RebalancerTestUtil.java
View File

@ -54,6 +54,7 @@ public class RebalancerTestUtil {
diskInfo.setPathHash(id + i);
diskInfo.setTotalCapacityB(totalCap);
diskInfo.setDataUsedCapacityB(usedCaps.get(i));
diskInfo.setAvailableCapacityB(totalCap - usedCaps.get(i));
disks.put(diskInfo.getRootPath(), diskInfo);
}
be.setDisks(ImmutableMap.copyOf(disks));