database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

Improve show column stats performance. (#31298)

Browse Source
This commit is contained in:
Jibing-Li
2024-02-23 10:44:13 +08:00
committed by yiguolei
parent 0bf13525d7
commit 04c295c4c2
4 changed files with 86 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
fe/fe-core/src/main/java/org/apache/doris/analysis/ShowColumnStatsStmt.java
View File

@ -180,4 +180,8 @@ public class ShowColumnStatsStmt extends ShowStmt {
public boolean isCached() {
return cached;
}
public boolean isAllColumns() {
return columnNames == null;
}
}

32
fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java
View File

@ -197,6 +197,7 @@ import org.apache.doris.mysql.privilege.PrivPredicate;
import org.apache.doris.statistics.AnalysisInfo;
import org.apache.doris.statistics.ColumnStatistic;
import org.apache.doris.statistics.Histogram;
import org.apache.doris.statistics.ResultRow;
import org.apache.doris.statistics.StatisticsRepository;
import org.apache.doris.statistics.TableStatsMeta;
import org.apache.doris.statistics.query.QueryStatsUtil;
@ -2553,7 +2554,35 @@ public class ShowExecutor {
Set<String> columnNames = showColumnStatsStmt.getColumnNames();
PartitionNames partitionNames = showColumnStatsStmt.getPartitionNames();
boolean showCache = showColumnStatsStmt.isCached();
boolean isAllColumns = showColumnStatsStmt.isAllColumns();
if (isAllColumns && !showCache && partitionNames == null) {
getStatsForAllColumns(columnStatistics, tableIf);
} else {
getStatsForSpecifiedColumns(columnStatistics, columnNames, tableIf, showCache, tableName, partitionNames);
}
resultSet = showColumnStatsStmt.constructResultSet(columnStatistics);
}
private void getStatsForAllColumns(List<Pair<Pair<String, String>, ColumnStatistic>> columnStatistics,
TableIf tableIf) throws AnalysisException {
List<ResultRow> resultRows = StatisticsRepository.queryColumnStatisticsForTable(tableIf.getId());
for (ResultRow row : resultRows) {
String indexName = "N/A";
long indexId = Long.parseLong(row.get(4));
if (indexId != -1) {
indexName = ((OlapTable) tableIf).getIndexNameById(indexId);
if (indexName == null) {
continue;
}
}
columnStatistics.add(Pair.of(Pair.of(row.get(5), indexName), ColumnStatistic.fromResultRow(row)));
}
}
private void getStatsForSpecifiedColumns(List<Pair<Pair<String, String>, ColumnStatistic>> columnStatistics,
Set<String> columnNames, TableIf tableIf, boolean showCache,
TableName tableName, PartitionNames partitionNames)
throws AnalysisException {
for (String colName : columnNames) {
// Olap base index use -1 as index id.
List<Long> indexIds = Lists.newArrayList();
@ -2584,13 +2613,12 @@ public class ShowExecutor {
} else {
String finalIndexName = indexName;
columnStatistics.addAll(StatisticsRepository.queryColumnStatisticsByPartitions(tableName,
colName, showColumnStatsStmt.getPartitionNames().getPartitionNames())
colName, partitionNames.getPartitionNames())
.stream().map(s -> Pair.of(Pair.of(colName, finalIndexName), s))
.collect(Collectors.toList()));
}
}
}
resultSet = showColumnStatsStmt.constructResultSet(columnStatistics);
}
public void handleShowColumnHist() {

13
fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsRepository.java
View File

@ -102,6 +102,11 @@ public class StatisticsRepository {
+ " ${inPredicate}"
+ " AND part_id IS NOT NULL";
private static final String FETCH_TABLE_STATISTICS = "SELECT * FROM "
+ FeConstants.INTERNAL_DB_NAME + "." + StatisticConstants.STATISTIC_TBL_NAME
+ " WHERE tbl_id = ${tblId}"
+ " AND part_id IS NULL";
public static ColumnStatistic queryColumnStatisticsByName(long tableId, long indexId, String colName) {
ResultRow resultRow = queryColumnStatisticById(tableId, indexId, colName);
if (resultRow == null) {
@ -126,6 +131,14 @@ public class StatisticsRepository {
Collectors.toList());
}
public static List<ResultRow> queryColumnStatisticsForTable(long tableId)
throws AnalysisException {
Map<String, String> params = new HashMap<>();
params.put("tblId", String.valueOf(tableId));
List<ResultRow> rows = StatisticsUtil.executeQuery(FETCH_TABLE_STATISTICS, params);
return rows == null ? Collections.emptyList() : rows;
}
public static ResultRow queryColumnStatisticById(long tblId, long indexId, String colName) {
return queryColumnStatisticById(tblId, indexId, colName, false);
}