database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[opt](fe) Optimize fe show table statistics (#35457) (#43858)

Browse Source
This commit is contained in:
Lei Zhang
2024-11-14 12:41:29 +08:00
committed by GitHub
parent 4531cd86e3
commit 4f2a67251a
5 changed files with 138 additions and 53 deletions
Download Patch File Download Diff File Expand all files Collapse all files

13
fe/fe-core/src/main/java/org/apache/doris/analysis/ShowDataStmt.java
View File

@ -178,14 +178,11 @@ public class ShowDataStmt extends ShowStmt {
long tableSize = 0;
long replicaCount = 0;
long remoteSize = 0;
olapTable.readLock();
try {
tableSize = olapTable.getDataSize();
replicaCount = olapTable.getReplicaCount();
remoteSize = olapTable.getRemoteDataSize();
} finally {
olapTable.readUnlock();
}
tableSize = olapTable.getDataSize();
replicaCount = olapTable.getReplicaCount();
remoteSize = olapTable.getRemoteDataSize();
//|TableName|Size|ReplicaCount|RemoteSize
List<Object> row = Arrays.asList(table.getName(), tableSize, replicaCount, remoteSize);
totalRowsObject.add(row);

21
fe/fe-core/src/main/java/org/apache/doris/catalog/Database.java
View File

@ -299,18 +299,14 @@ public class Database extends MetaObject implements Writable, DatabaseIf<Table>
}
OlapTable olapTable = (OlapTable) table;
olapTable.readLock();
try {
usedDataSize = usedDataSize + olapTable.getDataSize();
usedRemoteDataSize = usedRemoteDataSize + olapTable.getRemoteDataSize();
} finally {
olapTable.readUnlock();
}
usedDataSize = usedDataSize + olapTable.getDataSize();
usedRemoteDataSize = usedRemoteDataSize + olapTable.getRemoteDataSize();
}
return Pair.of(usedDataSize, usedRemoteDataSize);
}
public long getReplicaCountWithLock() {
public long getReplicaCount() {
readLock();
try {
long usedReplicaCount = 0;
@ -320,12 +316,7 @@ public class Database extends MetaObject implements Writable, DatabaseIf<Table>
}
OlapTable olapTable = (OlapTable) table;
olapTable.readLock();
try {
usedReplicaCount = usedReplicaCount + olapTable.getReplicaCount();
} finally {
olapTable.readUnlock();
}
usedReplicaCount = usedReplicaCount + olapTable.getReplicaCount();
}
return usedReplicaCount;
} finally {
@ -334,7 +325,7 @@ public class Database extends MetaObject implements Writable, DatabaseIf<Table>
}
public long getReplicaQuotaLeftWithLock() {
long leftReplicaQuota = replicaQuotaSize - getReplicaCountWithLock();
long leftReplicaQuota = replicaQuotaSize - getReplicaCount();
return Math.max(leftReplicaQuota, 0L);
}

119
fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java
View File

@ -87,6 +87,7 @@ import com.google.common.collect.Maps;
import com.google.common.collect.Range;
import com.google.common.collect.Sets;
import com.google.gson.annotations.SerializedName;
import lombok.Getter;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
@ -186,6 +187,8 @@ public class OlapTable extends Table implements MTMVRelatedTableIf {
private AutoIncrementGenerator autoIncrementGenerator;
private volatile Statistics statistics = new Statistics();
public OlapTable() {
// for persist
super(TableType.OLAP);
@ -1981,34 +1984,6 @@ public class OlapTable extends Table implements MTMVRelatedTableIf {
return oldPartition;
}
public long getDataSize(boolean singleReplica) {
long dataSize = 0;
for (Partition partition : getAllPartitions()) {
dataSize += partition.getDataSize(singleReplica);
}
return dataSize;
}
public long getDataSize() {
return getDataSize(false);
}
public long getRemoteDataSize() {
long remoteDataSize = 0;
for (Partition partition : getAllPartitions()) {
remoteDataSize += partition.getRemoteDataSize();
}
return remoteDataSize;
}
public long getReplicaCount() {
long replicaCount = 0;
for (Partition partition : getAllPartitions()) {
replicaCount += partition.getReplicaCount();
}
return replicaCount;
}
public void checkNormalStateForAlter() throws DdlException {
if (state != OlapTableState.NORMAL) {
throw new DdlException("Table[" + name + "]'s state(" + state.toString()
@ -3087,4 +3062,92 @@ public class OlapTable extends Table implements MTMVRelatedTableIf {
protected void addIndexNameToIdForUnitTest(String name, long id) {
indexNameToId.put(name, id);
}
public void setStatistics(Statistics statistics) {
this.statistics = statistics;
}
public static class Statistics {
@Getter
private String dbName;
@Getter
private String tableName;
@Getter
private Long dataSize; // single replica data size
@Getter
private Long totalReplicaDataSize;
@Getter
private Long remoteDataSize; // single replica remote data size
@Getter
private Long replicaCount;
@Getter
private Long rowCount;
@Getter
private Long rowsetCount;
@Getter
private Long segmentCount;
public Statistics() {
this.dbName = null;
this.tableName = null;
this.dataSize = 0L;
this.totalReplicaDataSize = 0L;
this.remoteDataSize = 0L;
this.replicaCount = 0L;
this.rowCount = 0L;
this.rowsetCount = 0L;
this.segmentCount = 0L;
}
public Statistics(String dbName, String tableName,
Long dataSize, Long totalReplicaDataSize,
Long remoteDataSize, Long replicaCount, Long rowCount,
Long rowsetCount, Long segmentCount) {
this.dbName = dbName;
this.tableName = tableName;
this.dataSize = dataSize;
this.totalReplicaDataSize = totalReplicaDataSize;
this.remoteDataSize = remoteDataSize;
this.replicaCount = replicaCount;
this.rowCount = rowCount;
this.rowsetCount = rowsetCount;
this.segmentCount = segmentCount;
}
}
public long getDataSize() {
return getDataSize(false);
}
public long getDataSize(boolean singleReplica) {
if (singleReplica) {
statistics.getDataSize();
}
return statistics.getTotalReplicaDataSize();
}
public long getRemoteDataSize() {
return statistics.getRemoteDataSize();
}
public long getReplicaCount() {
return statistics.getReplicaCount();
}
}

36
fe/fe-core/src/main/java/org/apache/doris/catalog/TabletStatMgr.java
View File

@ -104,6 +104,16 @@ public class TabletStatMgr extends MasterDaemon {
continue;
}
OlapTable olapTable = (OlapTable) table;
Long tableDataSize = 0L;
Long tableTotalReplicaDataSize = 0L;
Long tableRemoteDataSize = 0L;
Long tableReplicaCount = 0L;
Long tableRowCount = 0L;
// Use try write lock to avoid such cases
// Time1: Thread1 hold read lock for 5min
// Time2: Thread2 want to add write lock, then it will be the first element in lock queue
@ -120,7 +130,12 @@ public class TabletStatMgr extends MasterDaemon {
long indexRowCount = 0L;
boolean indexReported = true;
for (Tablet tablet : index.getTablets()) {
long tabletRowCount = Long.MAX_VALUE;
Long tabletDataSize = 0L;
Long tabletRemoteDataSize = 0L;
Long tabletRowCount = Long.MAX_VALUE;
boolean tabletReported = false;
for (Replica replica : tablet.getReplicas()) {
LOG.debug("Table {} replica {} current version {}, report version {}",
@ -146,12 +161,26 @@ public class TabletStatMgr extends MasterDaemon {
}
tabletRowCount = replica.getRowCount();
}
if (replica.getDataSize() > tabletDataSize) {
tabletDataSize = replica.getDataSize();
}
tableTotalReplicaDataSize += replica.getDataSize();
if (replica.getRemoteDataSize() > tabletRemoteDataSize) {
tabletRemoteDataSize = replica.getRemoteDataSize();
}
tableReplicaCount++;
}
tableDataSize += tabletDataSize;
tableRemoteDataSize += tabletRemoteDataSize;
// When all BEs are down, avoid set Long.MAX_VALUE to index and table row count. Use 0.
if (tabletRowCount == Long.MAX_VALUE) {
tabletRowCount = 0L;
}
tableRowCount += tabletRowCount;
indexRowCount += tabletRowCount;
// Only when all tablets of this index are reported, we set indexReported to true.
indexReported = indexReported && tabletReported;
@ -163,6 +192,11 @@ public class TabletStatMgr extends MasterDaemon {
indexReported, indexRowCount);
} // end for indices
} // end for partitions
olapTable.setStatistics(new OlapTable.Statistics(db.getName(), table.getName(),
tableDataSize, tableTotalReplicaDataSize,
tableRemoteDataSize, tableReplicaCount, tableRowCount, 0L, 0L));
if (LOG.isDebugEnabled()) {
LOG.debug("finished to set row num for table: {} in database: {}",
table.getName(), db.getFullName());

2
fe/fe-core/src/main/java/org/apache/doris/common/proc/DbsProcDir.java
View File

@ -113,7 +113,7 @@ public class DbsProcDir implements ProcDirInterface {
String readableQuota = DebugUtil.printByteWithUnit(dataQuota);
String lastCheckTime = (db instanceof Database) ? TimeUtils.longToTimeString(
((Database) db).getLastCheckTime()) : FeConstants.null_string;
long replicaCount = (db instanceof Database) ? ((Database) db).getReplicaCountWithLock() : 0;
long replicaCount = (db instanceof Database) ? ((Database) db).getReplicaCount() : 0;
long replicaQuota = (db instanceof Database) ? ((Database) db).getReplicaQuota() : 0;
long transactionQuota = (db instanceof Database) ? ((Database) db).getTransactionQuotaSize() : 0;
dbInfo.add(readableUsedQuota);