database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[feat](fe) Show db total size and recycle bin size with ShowDataStmt (#30663)

Browse Source
This commit is contained in:
Lei Zhang
2024-02-08 12:06:01 +08:00
committed by yiguolei
parent 3e07167897
commit cef31f6709
6 changed files with 223 additions and 11 deletions
Download Patch File Download Diff File Expand all files Collapse all files

88
fe/fe-core/src/main/java/org/apache/doris/analysis/ShowDataStmt.java
View File

@ -19,6 +19,7 @@ package org.apache.doris.analysis;
import org.apache.doris.catalog.Column;
import org.apache.doris.catalog.Database;
import org.apache.doris.catalog.DatabaseIf;
import org.apache.doris.catalog.Env;
import org.apache.doris.catalog.MaterializedIndex;
import org.apache.doris.catalog.OlapTable;
@ -38,6 +39,7 @@ import org.apache.doris.mysql.privilege.PrivPredicate;
import org.apache.doris.qe.ConnectContext;
import org.apache.doris.qe.ShowResultSetMetaData;
import com.google.common.base.Strings;
import com.google.common.collect.ComparisonChain;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.Lists;
@ -54,6 +56,16 @@ import java.util.TreeMap;
import java.util.TreeSet;
public class ShowDataStmt extends ShowStmt {
private static final ShowResultSetMetaData SHOW_DATABASE_DATA_META_DATA =
ShowResultSetMetaData.builder()
.addColumn(new Column("DbId", ScalarType.createVarchar(20)))
.addColumn(new Column("DbName", ScalarType.createVarchar(20)))
.addColumn(new Column("Size", ScalarType.createVarchar(30)))
.addColumn(new Column("RemoteSize", ScalarType.createVarchar(30)))
.addColumn(new Column("RecycleSize", ScalarType.createVarchar(30)))
.addColumn(new Column("RecycleRemoteSize", ScalarType.createVarchar(30)))
.build();
private static final ShowResultSetMetaData SHOW_TABLE_DATA_META_DATA =
ShowResultSetMetaData.builder()
.addColumn(new Column("TableName", ScalarType.createVarchar(20)))
@ -71,6 +83,11 @@ public class ShowDataStmt extends ShowStmt {
.addColumn(new Column("RowCount", ScalarType.createVarchar(20)))
.addColumn(new Column("RemoteSize", ScalarType.createVarchar(30)))
.build();
public static final ImmutableList<String> SHOW_DATABASE_DATA_META_DATA_ORIGIN =
new ImmutableList.Builder<String>().add("DbId").add("DbName").add("Size")
.add("RemoteSize").add("RecycleSize").add("RecycleRemoteSize").build();
public static final ImmutableList<String> SHOW_TABLE_DATA_META_DATA_ORIGIN =
new ImmutableList.Builder<String>().add("TableName").add("Size").add("ReplicaCount")
.add("RemoteSize").build();
@ -97,6 +114,10 @@ public class ShowDataStmt extends ShowStmt {
public void analyze(Analyzer analyzer) throws UserException {
super.analyze(analyzer);
dbName = analyzer.getDefaultDb();
if (Strings.isNullOrEmpty(dbName)) {
getAllDbStats();
return;
}
if (tableName != null) {
tableName.analyze(analyzer);
// disallow external catalog
@ -374,6 +395,9 @@ public class ShowDataStmt extends ShowStmt {
@Override
public ShowResultSetMetaData getMetaData() {
if (Strings.isNullOrEmpty(dbName)) {
return SHOW_DATABASE_DATA_META_DATA;
}
if (tableName != null) {
return SHOW_INDEX_DATA_META_DATA;
} else {
@ -408,4 +432,68 @@ public class ShowDataStmt extends ShowStmt {
public String toString() {
return toSql();
}
private void getAllDbStats() {
List<String> dbNames = Env.getCurrentInternalCatalog().getDbNames();
if (dbNames == null || dbNames.isEmpty()) {
return;
}
long totalSize = 0;
long totalRemoteSize = 0;
long totalRecycleSize = 0;
long totalRecycleRemoteSize = 0;
Map<Long, Pair<Long, Long>> dbToRecycleSize = Env.getCurrentRecycleBin().getDbToRecycleSize();
// show all database datasize
for (String dbName : dbNames) {
DatabaseIf db = Env.getCurrentInternalCatalog().getDbNullable(dbName);
if (db == null) {
continue;
}
List<String> dbInfo = new ArrayList<>();
db.readLock();
try {
dbInfo.add(String.valueOf(db.getId()));
dbInfo.add(dbName);
Pair<Long, Long> usedSize = ((Database) db).getUsedDataSize();
dbInfo.add(String.valueOf(usedSize.first));
dbInfo.add(String.valueOf(usedSize.second));
totalSize += usedSize.first;
totalRemoteSize += usedSize.second;
} finally {
db.readUnlock();
}
Pair<Long, Long> recycleSize = dbToRecycleSize.getOrDefault(db.getId(), Pair.of(0L, 0L));
dbInfo.add(String.valueOf(recycleSize.first));
dbInfo.add(String.valueOf(recycleSize.second));
totalRecycleSize += recycleSize.first;
totalRecycleRemoteSize += recycleSize.second;
dbToRecycleSize.remove(db.getId());
totalRows.add(dbInfo);
}
// Append left database in recycle bin
for (Map.Entry<Long, Pair<Long, Long>> entry : dbToRecycleSize.entrySet()) {
List<String> dbInfo = new ArrayList<>();
dbInfo.add(String.valueOf(entry.getKey()));
dbInfo.add("NULL");
dbInfo.add("0");
dbInfo.add("0");
dbInfo.add(String.valueOf(entry.getValue().first));
dbInfo.add(String.valueOf(entry.getValue().second));
totalRecycleSize += entry.getValue().first;
totalRecycleRemoteSize += entry.getValue().second;
totalRows.add(dbInfo);
}
// calc total size
List<String> dbInfo = new ArrayList<>();
dbInfo.add("Total");
dbInfo.add("NULL");
dbInfo.add(String.valueOf(totalSize));
dbInfo.add(String.valueOf(totalRemoteSize));
dbInfo.add(String.valueOf(totalRecycleSize));
dbInfo.add(String.valueOf(totalRecycleRemoteSize));
totalRows.add(dbInfo);
}
}

39
fe/fe-core/src/main/java/org/apache/doris/catalog/CatalogRecycleBin.java
View File

@ -1100,6 +1100,45 @@ public class CatalogRecycleBin extends MasterDaemon implements Writable {
return Stream.of(dbInfos, tableInfos, partitionInfos).flatMap(Collection::stream).collect(Collectors.toList());
}
public synchronized Map<Long, Pair<Long, Long>> getDbToRecycleSize() {
Map<Long, Pair<Long, Long>> dbToRecycleSize = new HashMap<>();
for (Map.Entry<Long, RecycleTableInfo> entry : idToTable.entrySet()) {
RecycleTableInfo tableInfo = entry.getValue();
Table table = tableInfo.getTable();
if (!(table instanceof OlapTable)) {
continue;
}
long dataSize = table.getDataSize(false);
long remoteDataSize = ((OlapTable) table).getRemoteDataSize();
dbToRecycleSize.compute(tableInfo.getDbId(), (k, v) -> {
if (v == null) {
return Pair.of(dataSize, remoteDataSize);
} else {
v.first += dataSize;
v.second += remoteDataSize;
return v;
}
});
}
for (Map.Entry<Long, RecyclePartitionInfo> entry : idToPartition.entrySet()) {
RecyclePartitionInfo partitionInfo = entry.getValue();
Partition partition = partitionInfo.getPartition();
long dataSize = partition.getDataSize(false);
long remoteDataSize = partition.getRemoteDataSize();
dbToRecycleSize.compute(partitionInfo.getDbId(), (k, v) -> {
if (v == null) {
return Pair.of(dataSize, remoteDataSize);
} else {
v.first += dataSize;
v.second += remoteDataSize;
return v;
}
});
}
return dbToRecycleSize;
}
// Need to add "synchronized", because when calling /dump api to dump image,
// this class is not protected by any lock, will throw ConcurrentModificationException.
@Override

20
fe/fe-core/src/main/java/org/apache/doris/catalog/Database.java
View File

@ -272,10 +272,19 @@ public class Database extends MetaObject implements Writable, DatabaseIf<Table>
}
public long getUsedDataQuotaWithLock() {
long usedDataQuota = 0;
return getUsedDataSize().first;
}
public Pair<Long, Long> getUsedDataSize() {
long usedDataSize = 0;
long usedRemoteDataSize = 0;
List<Table> tables = new ArrayList<>();
readLock();
List<Table> tables = new ArrayList<>(this.idToTable.values());
readUnlock();
try {
tables.addAll(this.idToTable.values());
} finally {
readUnlock();
}
for (Table table : tables) {
if (table.getType() != TableType.OLAP) {
@ -285,12 +294,13 @@ public class Database extends MetaObject implements Writable, DatabaseIf<Table>
OlapTable olapTable = (OlapTable) table;
olapTable.readLock();
try {
usedDataQuota = usedDataQuota + olapTable.getDataSize();
usedDataSize = usedDataSize + olapTable.getDataSize();
usedRemoteDataSize = usedRemoteDataSize + olapTable.getRemoteDataSize();
} finally {
olapTable.readUnlock();
}
}
return usedDataQuota;
return Pair.of(usedDataSize, usedRemoteDataSize);
}
public long getReplicaCountWithLock() {