database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[enhancement](show data skew) Support show avg_row_count for data skew of one table (#10790)

Browse Source
This commit is contained in:
caiconghui
2022-07-13 08:27:20 +08:00
committed by GitHub
parent 6063c0c9c8
commit d278f400d4
9 changed files with 141 additions and 16 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
fe/fe-core/src/main/java/org/apache/doris/analysis/ShowDataSkewStmt.java
View File

@ -34,7 +34,7 @@ import com.google.common.collect.ImmutableList;
// show data skew from tbl [partition(p1, p2, ...)]
public class ShowDataSkewStmt extends ShowStmt {
public static final ImmutableList<String> TITLE_NAMES = new ImmutableList.Builder<String>()
.add("BucketIdx").add("AvgDataSize")
.add("BucketIdx").add("AvgRowCount").add("AvgDataSize")
.add("Graph").add("Percent")
.build();

21
fe/fe-core/src/main/java/org/apache/doris/catalog/MetadataViewer.java
View File

@ -282,9 +282,11 @@ public class MetadataViewer {
break;
}
DistributionInfo distributionInfo = partition.getDistributionInfo();
List<Long> tabletInfos = Lists.newArrayListWithCapacity(distributionInfo.getBucketNum());
List<Long> rowCountTabletInfos = Lists.newArrayListWithCapacity(distributionInfo.getBucketNum());
List<Long> dataSizeTabletInfos = Lists.newArrayListWithCapacity(distributionInfo.getBucketNum());
for (long i = 0; i < distributionInfo.getBucketNum(); i++) {
tabletInfos.add(0L);
rowCountTabletInfos.add(0L);
dataSizeTabletInfos.add(0L);
}
long totalSize = 0;
@ -292,20 +294,23 @@ public class MetadataViewer {
List<Long> tabletIds = mIndex.getTabletIdsInOrder();
for (int i = 0; i < tabletIds.size(); i++) {
Tablet tablet = mIndex.getTablet(tabletIds.get(i));
long rowCount = tablet.getRowCount(true);
long dataSize = tablet.getDataSize(true);
tabletInfos.set(i, tabletInfos.get(i) + dataSize);
rowCountTabletInfos.set(i, rowCountTabletInfos.get(i) + rowCount);
dataSizeTabletInfos.set(i, dataSizeTabletInfos.get(i) + dataSize);
totalSize += dataSize;
}
}
// graph
for (int i = 0; i < tabletInfos.size(); i++) {
for (int i = 0; i < distributionInfo.getBucketNum(); i++) {
List<String> row = Lists.newArrayList();
row.add(String.valueOf(i));
row.add(tabletInfos.get(i).toString());
row.add(graph(tabletInfos.get(i), totalSize));
row.add(totalSize == tabletInfos.get(i)
? "100.00%" : df.format((double) tabletInfos.get(i) / totalSize));
row.add(rowCountTabletInfos.get(i).toString());
row.add(dataSizeTabletInfos.get(i).toString());
row.add(graph(dataSizeTabletInfos.get(i), totalSize));
row.add(totalSize == dataSizeTabletInfos.get(i)
? "100.00%" : df.format((double) dataSizeTabletInfos.get(i) / totalSize));
result.add(row);
}
} finally {

6
fe/fe-core/src/main/java/org/apache/doris/catalog/Tablet.java
View File

@ -394,6 +394,12 @@ public class Tablet extends MetaObject implements Writable {
return singleReplica ? Double.valueOf(s.average().orElse(0)).longValue() : s.sum();
}
public long getRowCount(boolean singleReplica) {
LongStream s = replicas.stream().filter(r -> r.getState() == ReplicaState.NORMAL)
.mapToLong(Replica::getRowCount);
return singleReplica ? Double.valueOf(s.average().orElse(0)).longValue() : s.sum();
}
/**
* A replica is healthy only if
* 1. the backend is available

10
fe/fe-core/src/main/java/org/apache/doris/common/proc/TabletHealthProcDir.java
View File

@ -290,11 +290,11 @@ public class TabletHealthProcDir implements ProcDirInterface {
this.colocateMismatchNum += other.colocateMismatchNum;
this.colocateRedundantNum += other.colocateRedundantNum;
this.needFurtherRepairNum += other.needFurtherRepairNum;
this.unrecoverableNum += unrecoverableNum;
this.replicaCompactionTooSlowNum += replicaCompactionTooSlowNum;
this.inconsistentNum += inconsistentNum;
this.oversizeNum += oversizeNum;
this.cloningNum += cloningNum;
this.unrecoverableNum += other.unrecoverableNum;
this.replicaCompactionTooSlowNum += other.replicaCompactionTooSlowNum;
this.inconsistentNum += other.inconsistentNum;
this.oversizeNum += other.oversizeNum;
this.cloningNum += other.cloningNum;
return this;
} else if (other.summary) {
return other.reduce(this);

4
fe/fe-core/src/test/java/org/apache/doris/analysis/AdminShowReplicaTest.java
View File

@ -64,7 +64,7 @@ public class AdminShowReplicaTest extends TestWithFeService {
executor = new ShowExecutor(connectContext, skewStmt);
resultSet = executor.execute();
Assert.assertEquals(10, resultSet.getResultRows().size());
Assert.assertEquals(4, resultSet.getResultRows().get(0).size());
Assert.assertEquals(5, resultSet.getResultRows().get(0).size());
// update tablets' data size and row count
Database db = Catalog.getCurrentInternalCatalog().getDbOrAnalysisException("default_cluster:test");
@ -88,7 +88,7 @@ public class AdminShowReplicaTest extends TestWithFeService {
resultSet = executor.execute();
Assert.assertEquals(10, resultSet.getResultRows().size());
Assert.assertEquals("4", resultSet.getResultRows().get(4).get(0));
Assert.assertEquals(4, resultSet.getResultRows().get(0).size());
Assert.assertEquals(5, resultSet.getResultRows().get(0).size());
}
@Test