database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[improvement](statistics)Return -1 to neredis if report olap table row count for new table is not done for all tablets. (#40457) (#40540)

Browse Source 
backport: https://github.com/apache/doris/pull/40457
This commit is contained in:
Jibing-Li
2024-09-10 12:55:53 +08:00
committed by GitHub
parent 6961c95eca
commit e43e6e2bba
14 changed files with 169 additions and 50 deletions
Download Patch File Download Diff File Expand all files Collapse all files

51
fe/fe-core/src/main/java/org/apache/doris/analysis/ShowTableStatsStmt.java
View File

@ -63,7 +63,9 @@ public class ShowTableStatsStmt extends ShowStmt {
new ImmutableList.Builder<String>()
.add("table_name")
.add("index_name")
.add("row_count")
.add("analyze_row_count")
.add("report_row_count")
.add("report_row_count_for_nereids")
.build();
private final TableName tableName;
@ -167,37 +169,33 @@ public class ShowTableStatsStmt extends ShowStmt {
return tableId;
}
public ShowResultSet constructResultSet(TableStatsMeta tableStatistic) {
public ShowResultSet constructResultSet(TableStatsMeta tableStatistic, TableIf table) {
if (indexName != null) {
return constructIndexResultSet(tableStatistic);
return constructIndexResultSet(tableStatistic, table);
}
return constructTableResultSet(tableStatistic);
return constructTableResultSet(tableStatistic, table);
}
public ShowResultSet constructEmptyResultSet() {
return new ShowResultSet(getMetaData(), new ArrayList<>());
}
public ShowResultSet constructResultSet(long rowCount) {
List<List<String>> result = Lists.newArrayList();
List<String> row = Lists.newArrayList();
row.add("");
row.add("");
row.add(String.valueOf(rowCount));
row.add("");
row.add("");
row.add("");
row.add("");
row.add("");
result.add(row);
return new ShowResultSet(getMetaData(), result);
}
public ShowResultSet constructTableResultSet(TableStatsMeta tableStatistic) {
DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss");
public ShowResultSet constructTableResultSet(TableStatsMeta tableStatistic, TableIf table) {
if (tableStatistic == null) {
return new ShowResultSet(getMetaData(), new ArrayList<>());
List<List<String>> result = Lists.newArrayList();
List<String> row = Lists.newArrayList();
row.add("");
row.add("");
row.add(String.valueOf(table.getCachedRowCount()));
row.add("");
row.add("");
row.add("");
row.add("");
row.add("");
result.add(row);
return new ShowResultSet(getMetaData(), result);
}
DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss");
List<List<String>> result = Lists.newArrayList();
List<String> row = Lists.newArrayList();
row.add(String.valueOf(tableStatistic.updatedRows));
@ -216,7 +214,7 @@ public class ShowTableStatsStmt extends ShowStmt {
return new ShowResultSet(getMetaData(), result);
}
public ShowResultSet constructIndexResultSet(TableStatsMeta tableStatistic) {
public ShowResultSet constructIndexResultSet(TableStatsMeta tableStatistic, TableIf table) {
List<List<String>> result = Lists.newArrayList();
if (!(table instanceof OlapTable)) {
return new ShowResultSet(getMetaData(), result);
@ -226,14 +224,13 @@ public class ShowTableStatsStmt extends ShowStmt {
if (indexId == null) {
throw new RuntimeException(String.format("Index %s not exist.", indexName));
}
long rowCount = tableStatistic.getRowCount(olapTable.getIndexIdByName(indexName));
if (rowCount == -1) {
return new ShowResultSet(getMetaData(), result);
}
long rowCount = tableStatistic == null ? -1 : tableStatistic.getRowCount(olapTable.getIndexIdByName(indexName));
List<String> row = Lists.newArrayList();
row.add(table.getName());
row.add(indexName);
row.add(String.valueOf(rowCount));
row.add(String.valueOf(olapTable.getRowCountForIndex(indexId, false)));
row.add(String.valueOf(olapTable.getRowCountForIndex(indexId, true)));
result.add(row);
return new ShowResultSet(getMetaData(), result);
}

10
fe/fe-core/src/main/java/org/apache/doris/catalog/MaterializedIndex.java
View File

@ -73,6 +73,8 @@ public class MaterializedIndex extends MetaObject implements Writable, GsonPostP
@SerializedName(value = "rollupFinishedVersion")
private long rollupFinishedVersion;
private boolean rowCountReported = false;
public MaterializedIndex() {
this.state = IndexState.NORMAL;
this.idToTablets = new HashMap<>();
@ -206,6 +208,14 @@ public class MaterializedIndex extends MetaObject implements Writable, GsonPostP
return -1;
}
public void setRowCountReported(boolean reported) {
this.rowCountReported = reported;
}
public boolean getRowCountReported() {
return this.rowCountReported;
}
@Override
public void write(DataOutput out) throws IOException {
super.write(out);

13
fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java
View File

@ -1396,18 +1396,17 @@ public class OlapTable extends Table implements MTMVRelatedTableIf {
@Override
public long fetchRowCount() {
long rowCount = 0;
for (Map.Entry<Long, Partition> entry : idToPartition.entrySet()) {
rowCount += entry.getValue().getBaseIndex().getRowCount();
}
return rowCount;
return getRowCountForIndex(baseIndexId, false);
}
public long getRowCountForIndex(long indexId) {
public long getRowCountForIndex(long indexId, boolean strict) {
long rowCount = 0;
for (Map.Entry<Long, Partition> entry : idToPartition.entrySet()) {
MaterializedIndex index = entry.getValue().getIndex(indexId);
rowCount += index == null ? 0 : index.getRowCount();
if (strict && !index.getRowCountReported()) {
return -1;
}
rowCount += (index == null || index.getRowCount() == -1) ? 0 : index.getRowCount();
}
return rowCount;
}

10
fe/fe-core/src/main/java/org/apache/doris/catalog/Replica.java
View File

@ -164,6 +164,8 @@ public class Replica implements Writable {
private long userDropTime = -1;
private long lastReportVersion = 0;
public Replica() {
}
@ -811,4 +813,12 @@ public class Replica implements Writable {
return Env.getCurrentSystemInfo().checkBackendScheduleAvailable(backendId)
&& !isUserDrop();
}
public void setLastReportVersion(long version) {
this.lastReportVersion = version;
}
public long getLastReportVersion() {
return lastReportVersion;
}
}

29
fe/fe-core/src/main/java/org/apache/doris/catalog/TabletStatMgr.java
View File

@ -118,17 +118,41 @@ public class TabletStatMgr extends MasterDaemon {
long version = partition.getVisibleVersion();
for (MaterializedIndex index : partition.getMaterializedIndices(IndexExtState.VISIBLE)) {
long indexRowCount = 0L;
boolean indexReported = true;
for (Tablet tablet : index.getTablets()) {
long tabletRowCount = 0L;
boolean tabletReported = false;
for (Replica replica : tablet.getReplicas()) {
LOG.debug("Table {} replica {} current version {}, report version {}",
olapTable.getName(), replica.getId(),
replica.getVersion(), replica.getLastReportVersion());
if (replica.checkVersionCatchUp(version, false)
&& replica.getRowCount() > tabletRowCount) {
&& replica.getRowCount() >= tabletRowCount) {
// 1. If replica version and reported replica version are all equal to
// PARTITION_INIT_VERSION, set tabletReported to true, which indicates this
// tablet is empty for sure when previous report.
// 2. If last report version is larger than PARTITION_INIT_VERSION, set
// tabletReported to true as well. That is, we only guarantee all replicas of
// the tablet are reported for the init version.
// e.g. When replica version is 2, but last reported version is 1,
// tabletReported would be false.
if (replica.getVersion() == Partition.PARTITION_INIT_VERSION
&& replica.getLastReportVersion() == Partition.PARTITION_INIT_VERSION
|| replica.getLastReportVersion() > Partition.PARTITION_INIT_VERSION) {
tabletReported = true;
}
tabletRowCount = replica.getRowCount();
}
}
indexRowCount += tabletRowCount;
// Only when all tablets of this index are reported, we set indexReported to true.
indexReported = indexReported && tabletReported;
} // end for tablets
index.setRowCountReported(indexReported);
index.setRowCount(indexRowCount);
LOG.debug("Table {} index {} all tablets reported[{}], row count {}",
olapTable.getName(), olapTable.getIndexNameById(index.getId()),
indexReported, indexRowCount);
} // end for indices
} // end for partitions
if (LOG.isDebugEnabled()) {
@ -157,6 +181,9 @@ public class TabletStatMgr extends MasterDaemon {
replica.setTotalVersionCount(stat.getTotalVersionCount());
replica.setVisibleVersionCount(stat.isSetVisibleVersionCount() ? stat.getVisibleVersionCount()
: stat.getTotalVersionCount());
// Older version BE doesn't set visible version. Set it to max for compatibility.
replica.setLastReportVersion(stat.isSetVisibleVersion() ? stat.getVisibleVersion()
: Long.MAX_VALUE);
}
}
}

11
fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java
View File

@ -2520,19 +2520,12 @@ public class ShowExecutor {
if (tableStats == null) {
resultSet = showTableStatsStmt.constructEmptyResultSet();
} else {
resultSet = showTableStatsStmt.constructResultSet(tableStats);
resultSet = showTableStatsStmt.constructResultSet(tableStats, tableIf);
}
return;
}
TableStatsMeta tableStats = Env.getCurrentEnv().getAnalysisManager().findTableStatsStatus(tableIf.getId());
/*
tableStats == null means it's not analyzed, in this case show the estimated row count.
*/
if (tableStats == null) {
resultSet = showTableStatsStmt.constructResultSet(tableIf.getCachedRowCount());
} else {
resultSet = showTableStatsStmt.constructResultSet(tableStats);
}
resultSet = showTableStatsStmt.constructResultSet(tableStats, tableIf);
}
private void handleShowColumnStats() throws AnalysisException {

6
fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisManager.java
View File

@ -640,7 +640,7 @@ public class AnalysisManager implements Writable {
if (tableStats == null) {
return;
}
invalidateLocalStats(catalogId, dbId, tblId, cols, tableStats);
invalidateLocalStats(catalogId, dbId, tblId, dropStatsStmt.isAllColumns() ? null : cols, tableStats);
// Drop stats ddl is master only operation.
invalidateRemoteStats(catalogId, dbId, tblId, cols, dropStatsStmt.isAllColumns());
StatisticsRepository.dropStatisticsByColNames(catalogId, dbId, tblId, cols);
@ -655,7 +655,7 @@ public class AnalysisManager implements Writable {
long dbId = table.getDatabase().getId();
long tableId = table.getId();
Set<String> cols = table.getSchemaAllIndexes(false).stream().map(Column::getName).collect(Collectors.toSet());
invalidateLocalStats(catalogId, dbId, tableId, cols, tableStats);
invalidateLocalStats(catalogId, dbId, tableId, null, tableStats);
// Drop stats ddl is master only operation.
invalidateRemoteStats(catalogId, dbId, tableId, cols, true);
StatisticsRepository.dropStatisticsByColNames(catalogId, dbId, table.getId(), cols);
@ -717,6 +717,8 @@ public class AnalysisManager implements Writable {
// To remove stale column name that is changed before.
if (allColumn) {
tableStats.removeAllColumn();
tableStats.clearIndexesRowCount();
removeTableStats(tableId);
}
tableStats.updatedTime = 0;
tableStats.userInjected = false;

2
fe/fe-core/src/main/java/org/apache/doris/statistics/OlapAnalysisTask.java
View File

@ -102,7 +102,7 @@ public class OlapAnalysisTask extends BaseAnalysisTask {
List<Long> tabletIds = pair.first;
long totalRowCount = info.indexId == -1
? tbl.getRowCount()
: ((OlapTable) tbl).getRowCountForIndex(info.indexId);
: ((OlapTable) tbl).getRowCountForIndex(info.indexId, false);
double scaleFactor = (double) totalRowCount / (double) pair.second;
// might happen if row count in fe metadata hasn't been updated yet
if (Double.isInfinite(scaleFactor) || Double.isNaN(scaleFactor)) {

6
fe/fe-core/src/main/java/org/apache/doris/statistics/TableStatsMeta.java
View File

@ -95,7 +95,7 @@ public class TableStatsMeta implements Writable, GsonPostProcessable {
public boolean userInjected;
@SerializedName("irc")
public ConcurrentMap<Long, Long> indexesRowCount = new ConcurrentHashMap<>();
private ConcurrentMap<Long, Long> indexesRowCount = new ConcurrentHashMap<>();
@VisibleForTesting
public TableStatsMeta() {
@ -212,6 +212,10 @@ public class TableStatsMeta implements Writable, GsonPostProcessable {
return indexesRowCount.getOrDefault(indexId, -1L);
}
public void clearIndexesRowCount() {
indexesRowCount.clear();
}
private void clearStaleIndexRowCount(OlapTable table) {
Iterator<Long> iterator = indexesRowCount.keySet().iterator();
List<Long> indexIds = table.getIndexIds();

3
fe/fe-core/src/main/java/org/apache/doris/transaction/DatabaseTransactionMgr.java
View File

@ -2102,6 +2102,9 @@ public class DatabaseTransactionMgr {
}
}
replica.updateVersionWithFailed(newVersion, lastFailedVersion, lastSuccessVersion);
if (newVersion == Partition.PARTITION_INIT_VERSION + 1) {
index.setRowCountReported(false);
}
Set<Long> partitionIds = backendPartitions.get(replica.getBackendId());
if (partitionIds == null) {
partitionIds = Sets.newHashSet();