database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[improve](statistics)Clean expired TableStatsMeta. (#39779) (#40253)

Browse Source 
backport: https://github.com/apache/doris/pull/39779
This commit is contained in:
Jibing-Li
2024-09-02 22:13:42 +08:00
committed by GitHub
parent 4a91b2d46b
commit a5efdc6085
8 changed files with 235 additions and 10 deletions
Download Patch File Download Diff File Expand all files Collapse all files

39
fe/fe-core/src/main/java/org/apache/doris/analysis/ShowTableStatsStmt.java
View File

@ -68,16 +68,26 @@ public class ShowTableStatsStmt extends ShowStmt {
private final TableName tableName;
private final PartitionNames partitionNames;
private final boolean cached;
private final String indexName;
private final long tableId;
private final boolean useTableId;
private TableIf table;
public ShowTableStatsStmt(TableName tableName, PartitionNames partitionNames, boolean cached, String indexName) {
public ShowTableStatsStmt(long tableId) {
this.tableName = null;
this.partitionNames = null;
this.indexName = null;
this.tableId = tableId;
this.useTableId = true;
}
public ShowTableStatsStmt(TableName tableName, PartitionNames partitionNames, String indexName) {
this.tableName = tableName;
this.partitionNames = partitionNames;
this.cached = cached;
this.indexName = indexName;
this.tableId = -1;
this.useTableId = false;
}
public TableName getTableName() {
@ -87,6 +97,13 @@ public class ShowTableStatsStmt extends ShowStmt {
@Override
public void analyze(Analyzer analyzer) throws UserException {
super.analyze(analyzer);
if (useTableId) {
if (!Env.getCurrentEnv().getAccessManager().checkGlobalPriv(ConnectContext.get(), PrivPredicate.SHOW)) {
ErrorReport.reportAnalysisException(ErrorCode.ERR_TABLEACCESS_DENIED_ERROR, "Permission denied",
ConnectContext.get().getQualifiedUser(), ConnectContext.get().getRemoteIP());
}
return;
}
tableName.analyze(analyzer);
if (partitionNames != null) {
partitionNames.analyze(analyzer);
@ -142,6 +159,14 @@ public class ShowTableStatsStmt extends ShowStmt {
return table;
}
public boolean isUseTableId() {
return useTableId;
}
public long getTableId() {
return tableId;
}
public ShowResultSet constructResultSet(TableStatsMeta tableStatistic) {
if (indexName != null) {
return constructIndexResultSet(tableStatistic);
@ -149,6 +174,10 @@ public class ShowTableStatsStmt extends ShowStmt {
return constructTableResultSet(tableStatistic);
}
public ShowResultSet constructEmptyResultSet() {
return new ShowResultSet(getMetaData(), new ArrayList<>());
}
public ShowResultSet constructResultSet(long rowCount) {
List<List<String>> result = Lists.newArrayList();
List<String> row = Lists.newArrayList();
@ -208,8 +237,4 @@ public class ShowTableStatsStmt extends ShowStmt {
result.add(row);
return new ShowResultSet(getMetaData(), result);
}
public boolean isCached() {
return cached;
}
}

11
fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java
View File

@ -2512,6 +2512,17 @@ public class ShowExecutor {
private void handleShowTableStats() {
ShowTableStatsStmt showTableStatsStmt = (ShowTableStatsStmt) stmt;
TableIf tableIf = showTableStatsStmt.getTable();
// Handle use table id to show table stats. Mainly for online debug.
if (showTableStatsStmt.isUseTableId()) {
long tableId = showTableStatsStmt.getTableId();
TableStatsMeta tableStats = Env.getCurrentEnv().getAnalysisManager().findTableStatsStatus(tableId);
if (tableStats == null) {
resultSet = showTableStatsStmt.constructEmptyResultSet();
} else {
resultSet = showTableStatsStmt.constructResultSet(tableStats);
}
return;
}
TableStatsMeta tableStats = Env.getCurrentEnv().getAnalysisManager().findTableStatsStatus(tableIf.getId());
/*
tableStats == null means it's not analyzed, in this case show the estimated row count.

5
fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisManager.java
View File

@ -84,6 +84,7 @@ import java.util.Collection;
import java.util.Collections;
import java.util.Comparator;
import java.util.HashMap;
import java.util.HashSet;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
@ -1103,6 +1104,10 @@ public class AnalysisManager implements Writable {
}
}
public Set<Long> getIdToTblStatsKeys() {
return new HashSet<>(idToTblStats.keySet());
}
public ColStatsMeta findColStatsMeta(long tblId, String indexName, String colName) {
TableStatsMeta tableStats = findTableStatsStatus(tblId);
if (tableStats == null) {

56
fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsCleaner.java
View File

@ -17,6 +17,7 @@
package org.apache.doris.statistics;
import org.apache.doris.catalog.Database;
import org.apache.doris.catalog.DatabaseIf;
import org.apache.doris.catalog.Env;
import org.apache.doris.catalog.MaterializedIndexMeta;
@ -27,6 +28,7 @@ import org.apache.doris.common.FeConstants;
import org.apache.doris.common.util.MasterDaemon;
import org.apache.doris.datasource.CatalogIf;
import org.apache.doris.datasource.InternalCatalog;
import org.apache.doris.persist.TableStatsDeletionLog;
import org.apache.doris.statistics.util.StatisticsUtil;
import com.google.common.collect.Maps;
@ -74,6 +76,7 @@ public class StatisticsCleaner extends MasterDaemon {
}
public synchronized void clear() {
clearTableStats();
try {
if (!init()) {
return;
@ -100,6 +103,59 @@ public class StatisticsCleaner extends MasterDaemon {
} while (!expiredStats.isEmpty());
}
private void clearTableStats() {
AnalysisManager analysisManager = Env.getCurrentEnv().getAnalysisManager();
Set<Long> tableIds = analysisManager.getIdToTblStatsKeys();
InternalCatalog internalCatalog = Env.getCurrentInternalCatalog();
for (long id : tableIds) {
try {
TableStatsMeta stats = analysisManager.findTableStatsStatus(id);
if (stats == null) {
continue;
}
// If ctlName, dbName and tblName exist, it means the table stats is created under new version.
// First try to find the table by the given names. If table exists, means the tableMeta is valid,
// it should be kept in memory.
try {
StatisticsUtil.findTable(stats.ctlName, stats.dbName, stats.tblName);
continue;
} catch (Exception e) {
LOG.debug("Table {}.{}.{} not found.", stats.ctlName, stats.dbName, stats.tblName);
}
// If we couldn't find table by names, try to find it in internal catalog. This is to support older
// version which the tableStats object doesn't store the names but only table id.
// We may remove external table's tableStats here, but it's not a big problem.
// Because the stats in column_statistics table is still available,
// the only disadvantage is auto analyze may be triggered for this table.
// But it only happens once, the new table stats object will have all the catalog, db and table names.
if (tableExistInInternalCatalog(internalCatalog, id)) {
continue;
}
LOG.info("Table {}.{}.{} with id {} not exist, remove its table stats record.",
stats.ctlName, stats.dbName, stats.tblName, id);
analysisManager.removeTableStats(id);
Env.getCurrentEnv().getEditLog().logDeleteTableStats(new TableStatsDeletionLog(id));
} catch (Exception e) {
LOG.info(e);
}
}
}
private boolean tableExistInInternalCatalog(InternalCatalog internalCatalog, long tableId) {
List<Long> dbIds = internalCatalog.getDbIds();
for (long dbId : dbIds) {
Database database = internalCatalog.getDbNullable(dbId);
if (database == null) {
continue;
}
TableIf table = database.getTableNullable(tableId);
if (table != null) {
return true;
}
}
return false;
}
private boolean init() {
try {
String dbName = FeConstants.INTERNAL_DB_NAME;

25
fe/fe-core/src/main/java/org/apache/doris/statistics/TableStatsMeta.java
View File

@ -44,9 +44,24 @@ import java.util.stream.Collectors;
public class TableStatsMeta implements Writable, GsonPostProcessable {
@SerializedName("ctlId")
public final long ctlId;
@SerializedName("ctln")
public final String ctlName;
@SerializedName("dbId")
public final long dbId;
@SerializedName("dbn")
public final String dbName;
@SerializedName("tblId")
public final long tblId;
@SerializedName("tbln")
public final String tblName;
@SerializedName("idxId")
public final long idxId;
@SerializedName("updatedRows")
@ -84,14 +99,24 @@ public class TableStatsMeta implements Writable, GsonPostProcessable {
@VisibleForTesting
public TableStatsMeta() {
ctlId = 0;
ctlName = null;
dbId = 0;
dbName = null;
tblId = 0;
tblName = null;
idxId = 0;
}
// It's necessary to store these fields separately from AnalysisInfo, since the lifecycle between AnalysisInfo
// and TableStats is quite different.
public TableStatsMeta(long rowCount, AnalysisInfo analyzedJob, TableIf table) {
this.ctlId = table.getDatabase().getCatalog().getId();
this.ctlName = table.getDatabase().getCatalog().getName();
this.dbId = table.getDatabase().getId();
this.dbName = table.getDatabase().getFullName();
this.tblId = table.getId();
this.tblName = table.getName();
this.idxId = -1;
this.rowCount = rowCount;
update(analyzedJob, table);