database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

Revert "[improvement](statistics)Return -1 to neredis if report olap table row count for new table is not done for all tablets. (#40457)" (#40616)

Browse Source 
Reverts apache/doris#40540
This commit is contained in:
yiguolei
2024-09-10 17:17:13 +08:00
committed by GitHub
parent 6f239ebcdd
commit 4e453dc1bb
14 changed files with 51 additions and 170 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
be/src/olap/tablet_manager.cpp
View File

@ -1035,7 +1035,6 @@ void TabletManager::build_all_report_tablets_info(std::map<TTabletId, TTablet>*
t_tablet_stat.__set_row_count(tablet_info.row_count);
t_tablet_stat.__set_total_version_count(tablet_info.total_version_count);
t_tablet_stat.__set_visible_version_count(tablet_info.visible_version_count);
t_tablet_stat.__set_visible_version(tablet_info.version);
};
for_each_tablet(handler, filter_all_tablets);

53
fe/fe-core/src/main/java/org/apache/doris/analysis/ShowTableStatsStmt.java
View File

@ -63,9 +63,7 @@ public class ShowTableStatsStmt extends ShowStmt {
new ImmutableList.Builder<String>()
.add("table_name")
.add("index_name")
.add("analyze_row_count")
.add("report_row_count")
.add("report_row_count_for_nereids")
.add("row_count")
.build();
private final TableName tableName;
@ -169,33 +167,37 @@ public class ShowTableStatsStmt extends ShowStmt {
return tableId;
}
public ShowResultSet constructResultSet(TableStatsMeta tableStatistic, TableIf table) {
public ShowResultSet constructResultSet(TableStatsMeta tableStatistic) {
if (indexName != null) {
return constructIndexResultSet(tableStatistic, table);
return constructIndexResultSet(tableStatistic);
}
return constructTableResultSet(tableStatistic, table);
return constructTableResultSet(tableStatistic);
}
public ShowResultSet constructEmptyResultSet() {
return new ShowResultSet(getMetaData(), new ArrayList<>());
}
public ShowResultSet constructTableResultSet(TableStatsMeta tableStatistic, TableIf table) {
if (tableStatistic == null) {
List<List<String>> result = Lists.newArrayList();
List<String> row = Lists.newArrayList();
row.add("");
row.add("");
row.add(String.valueOf(table.getCachedRowCount()));
row.add("");
row.add("");
row.add("");
row.add("");
row.add("");
result.add(row);
return new ShowResultSet(getMetaData(), result);
}
public ShowResultSet constructResultSet(long rowCount) {
List<List<String>> result = Lists.newArrayList();
List<String> row = Lists.newArrayList();
row.add("");
row.add("");
row.add(String.valueOf(rowCount));
row.add("");
row.add("");
row.add("");
row.add("");
row.add("");
result.add(row);
return new ShowResultSet(getMetaData(), result);
}
public ShowResultSet constructTableResultSet(TableStatsMeta tableStatistic) {
DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss");
if (tableStatistic == null) {
return new ShowResultSet(getMetaData(), new ArrayList<>());
}
List<List<String>> result = Lists.newArrayList();
List<String> row = Lists.newArrayList();
row.add(String.valueOf(tableStatistic.updatedRows));
@ -214,7 +216,7 @@ public class ShowTableStatsStmt extends ShowStmt {
return new ShowResultSet(getMetaData(), result);
}
public ShowResultSet constructIndexResultSet(TableStatsMeta tableStatistic, TableIf table) {
public ShowResultSet constructIndexResultSet(TableStatsMeta tableStatistic) {
List<List<String>> result = Lists.newArrayList();
if (!(table instanceof OlapTable)) {
return new ShowResultSet(getMetaData(), result);
@ -224,13 +226,14 @@ public class ShowTableStatsStmt extends ShowStmt {
if (indexId == null) {
throw new RuntimeException(String.format("Index %s not exist.", indexName));
}
long rowCount = tableStatistic == null ? -1 : tableStatistic.getRowCount(olapTable.getIndexIdByName(indexName));
long rowCount = tableStatistic.getRowCount(olapTable.getIndexIdByName(indexName));
if (rowCount == -1) {
return new ShowResultSet(getMetaData(), result);
}
List<String> row = Lists.newArrayList();
row.add(table.getName());
row.add(indexName);
row.add(String.valueOf(rowCount));
row.add(String.valueOf(olapTable.getRowCountForIndex(indexId, false)));
row.add(String.valueOf(olapTable.getRowCountForIndex(indexId, true)));
result.add(row);
return new ShowResultSet(getMetaData(), result);
}

10
fe/fe-core/src/main/java/org/apache/doris/catalog/MaterializedIndex.java
View File

@ -73,8 +73,6 @@ public class MaterializedIndex extends MetaObject implements Writable, GsonPostP
@SerializedName(value = "rollupFinishedVersion")
private long rollupFinishedVersion;
private boolean rowCountReported = false;
public MaterializedIndex() {
this.state = IndexState.NORMAL;
this.idToTablets = new HashMap<>();
@ -208,14 +206,6 @@ public class MaterializedIndex extends MetaObject implements Writable, GsonPostP
return -1;
}
public void setRowCountReported(boolean reported) {
this.rowCountReported = reported;
}
public boolean getRowCountReported() {
return this.rowCountReported;
}
@Override
public void write(DataOutput out) throws IOException {
super.write(out);

13
fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java
View File

@ -1396,17 +1396,18 @@ public class OlapTable extends Table implements MTMVRelatedTableIf {
@Override
public long fetchRowCount() {
return getRowCountForIndex(baseIndexId, false);
long rowCount = 0;
for (Map.Entry<Long, Partition> entry : idToPartition.entrySet()) {
rowCount += entry.getValue().getBaseIndex().getRowCount();
}
return rowCount;
}
public long getRowCountForIndex(long indexId, boolean strict) {
public long getRowCountForIndex(long indexId) {
long rowCount = 0;
for (Map.Entry<Long, Partition> entry : idToPartition.entrySet()) {
MaterializedIndex index = entry.getValue().getIndex(indexId);
if (strict && !index.getRowCountReported()) {
return -1;
}
rowCount += (index == null || index.getRowCount() == -1) ? 0 : index.getRowCount();
rowCount += index == null ? 0 : index.getRowCount();
}
return rowCount;
}

10
fe/fe-core/src/main/java/org/apache/doris/catalog/Replica.java
View File

@ -164,8 +164,6 @@ public class Replica implements Writable {
private long userDropTime = -1;
private long lastReportVersion = 0;
public Replica() {
}
@ -813,12 +811,4 @@ public class Replica implements Writable {
return Env.getCurrentSystemInfo().checkBackendScheduleAvailable(backendId)
&& !isUserDrop();
}
public void setLastReportVersion(long version) {
this.lastReportVersion = version;
}
public long getLastReportVersion() {
return lastReportVersion;
}
}

29
fe/fe-core/src/main/java/org/apache/doris/catalog/TabletStatMgr.java
View File

@ -118,41 +118,17 @@ public class TabletStatMgr extends MasterDaemon {
long version = partition.getVisibleVersion();
for (MaterializedIndex index : partition.getMaterializedIndices(IndexExtState.VISIBLE)) {
long indexRowCount = 0L;
boolean indexReported = true;
for (Tablet tablet : index.getTablets()) {
long tabletRowCount = 0L;
boolean tabletReported = false;
for (Replica replica : tablet.getReplicas()) {
LOG.debug("Table {} replica {} current version {}, report version {}",
olapTable.getName(), replica.getId(),
replica.getVersion(), replica.getLastReportVersion());
if (replica.checkVersionCatchUp(version, false)
&& replica.getRowCount() >= tabletRowCount) {
// 1. If replica version and reported replica version are all equal to
// PARTITION_INIT_VERSION, set tabletReported to true, which indicates this
// tablet is empty for sure when previous report.
// 2. If last report version is larger than PARTITION_INIT_VERSION, set
// tabletReported to true as well. That is, we only guarantee all replicas of
// the tablet are reported for the init version.
// e.g. When replica version is 2, but last reported version is 1,
// tabletReported would be false.
if (replica.getVersion() == Partition.PARTITION_INIT_VERSION
&& replica.getLastReportVersion() == Partition.PARTITION_INIT_VERSION
|| replica.getLastReportVersion() > Partition.PARTITION_INIT_VERSION) {
tabletReported = true;
}
&& replica.getRowCount() > tabletRowCount) {
tabletRowCount = replica.getRowCount();
}
}
indexRowCount += tabletRowCount;
// Only when all tablets of this index are reported, we set indexReported to true.
indexReported = indexReported && tabletReported;
} // end for tablets
index.setRowCountReported(indexReported);
index.setRowCount(indexRowCount);
LOG.debug("Table {} index {} all tablets reported[{}], row count {}",
olapTable.getName(), olapTable.getIndexNameById(index.getId()),
indexReported, indexRowCount);
} // end for indices
} // end for partitions
if (LOG.isDebugEnabled()) {
@ -181,9 +157,6 @@ public class TabletStatMgr extends MasterDaemon {
replica.setTotalVersionCount(stat.getTotalVersionCount());
replica.setVisibleVersionCount(stat.isSetVisibleVersionCount() ? stat.getVisibleVersionCount()
: stat.getTotalVersionCount());
// Older version BE doesn't set visible version. Set it to max for compatibility.
replica.setLastReportVersion(stat.isSetVisibleVersion() ? stat.getVisibleVersion()
: Long.MAX_VALUE);
}
}
}

11
fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java
View File

@ -2520,12 +2520,19 @@ public class ShowExecutor {
if (tableStats == null) {
resultSet = showTableStatsStmt.constructEmptyResultSet();
} else {
resultSet = showTableStatsStmt.constructResultSet(tableStats, tableIf);
resultSet = showTableStatsStmt.constructResultSet(tableStats);
}
return;
}
TableStatsMeta tableStats = Env.getCurrentEnv().getAnalysisManager().findTableStatsStatus(tableIf.getId());
resultSet = showTableStatsStmt.constructResultSet(tableStats, tableIf);
/*
tableStats == null means it's not analyzed, in this case show the estimated row count.
*/
if (tableStats == null) {
resultSet = showTableStatsStmt.constructResultSet(tableIf.getCachedRowCount());
} else {
resultSet = showTableStatsStmt.constructResultSet(tableStats);
}
}
private void handleShowColumnStats() throws AnalysisException {

6
fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisManager.java
View File

@ -640,7 +640,7 @@ public class AnalysisManager implements Writable {
if (tableStats == null) {
return;
}
invalidateLocalStats(catalogId, dbId, tblId, dropStatsStmt.isAllColumns() ? null : cols, tableStats);
invalidateLocalStats(catalogId, dbId, tblId, cols, tableStats);
// Drop stats ddl is master only operation.
invalidateRemoteStats(catalogId, dbId, tblId, cols, dropStatsStmt.isAllColumns());
StatisticsRepository.dropStatisticsByColNames(catalogId, dbId, tblId, cols);
@ -655,7 +655,7 @@ public class AnalysisManager implements Writable {
long dbId = table.getDatabase().getId();
long tableId = table.getId();
Set<String> cols = table.getSchemaAllIndexes(false).stream().map(Column::getName).collect(Collectors.toSet());
invalidateLocalStats(catalogId, dbId, tableId, null, tableStats);
invalidateLocalStats(catalogId, dbId, tableId, cols, tableStats);
// Drop stats ddl is master only operation.
invalidateRemoteStats(catalogId, dbId, tableId, cols, true);
StatisticsRepository.dropStatisticsByColNames(catalogId, dbId, table.getId(), cols);
@ -717,8 +717,6 @@ public class AnalysisManager implements Writable {
// To remove stale column name that is changed before.
if (allColumn) {
tableStats.removeAllColumn();
tableStats.clearIndexesRowCount();
removeTableStats(tableId);
}
tableStats.updatedTime = 0;
tableStats.userInjected = false;

2
fe/fe-core/src/main/java/org/apache/doris/statistics/OlapAnalysisTask.java
View File

@ -102,7 +102,7 @@ public class OlapAnalysisTask extends BaseAnalysisTask {
List<Long> tabletIds = pair.first;
long totalRowCount = info.indexId == -1
? tbl.getRowCount()
: ((OlapTable) tbl).getRowCountForIndex(info.indexId, false);
: ((OlapTable) tbl).getRowCountForIndex(info.indexId);
double scaleFactor = (double) totalRowCount / (double) pair.second;
// might happen if row count in fe metadata hasn't been updated yet
if (Double.isInfinite(scaleFactor) || Double.isNaN(scaleFactor)) {

6
fe/fe-core/src/main/java/org/apache/doris/statistics/TableStatsMeta.java
View File

@ -95,7 +95,7 @@ public class TableStatsMeta implements Writable, GsonPostProcessable {
public boolean userInjected;
@SerializedName("irc")
private ConcurrentMap<Long, Long> indexesRowCount = new ConcurrentHashMap<>();
public ConcurrentMap<Long, Long> indexesRowCount = new ConcurrentHashMap<>();
@VisibleForTesting
public TableStatsMeta() {
@ -212,10 +212,6 @@ public class TableStatsMeta implements Writable, GsonPostProcessable {
return indexesRowCount.getOrDefault(indexId, -1L);
}
public void clearIndexesRowCount() {
indexesRowCount.clear();
}
private void clearStaleIndexRowCount(OlapTable table) {
Iterator<Long> iterator = indexesRowCount.keySet().iterator();
List<Long> indexIds = table.getIndexIds();

3
fe/fe-core/src/main/java/org/apache/doris/transaction/DatabaseTransactionMgr.java
View File

@ -2102,9 +2102,6 @@ public class DatabaseTransactionMgr {
}
}
replica.updateVersionWithFailed(newVersion, lastFailedVersion, lastSuccessVersion);
if (newVersion == Partition.PARTITION_INIT_VERSION + 1) {
index.setRowCountReported(false);
}
Set<Long> partitionIds = backendPartitions.get(replica.getBackendId());
if (partitionIds == null) {
partitionIds = Sets.newHashSet();

1
gensrc/thrift/BackendService.thrift
View File

@ -37,7 +37,6 @@ struct TTabletStat {
4: optional i64 total_version_count
5: optional i64 remote_data_size
6: optional i64 visible_version_count
7: optional i64 visible_version
}
struct TTabletStatResult {

2
regression-test/suites/statistics/analyze_stats.groovy
View File

@ -2753,7 +2753,7 @@ PARTITION `p599` VALUES IN (599)
assertEquals("true", alter_result[0][7])
sql """drop stats alter_test"""
alter_result = sql """show table stats alter_test"""
assertEquals("", alter_result[0][7])
assertEquals("false", alter_result[0][7])
sql """alter table alter_test modify column id set stats ('row_count'='100', 'ndv'='0', 'num_nulls'='0.0', 'data_size'='2.69975443E8', 'min_value'='1', 'max_value'='2');"""
alter_result = sql """show column stats alter_test(id)"""
logger.info("show column alter_test(id) stats: " + alter_result)

74
regression-test/suites/statistics/test_analyze_mv.groovy
View File

@ -126,17 +126,6 @@ suite("test_analyze_mv") {
"replication_num" = "1"
)
"""
def result_row
if (!isCloudMode()) {
// Test row count report and report for nereids
result_row = sql """show index stats mvTestDup mvTestDup"""
assertEquals(1, result_row.size())
assertEquals("mvTestDup", result_row[0][0])
assertEquals("mvTestDup", result_row[0][1])
assertEquals("0", result_row[0][3])
assertEquals("-1", result_row[0][4])
}
createMV("create materialized view mv1 as select key1 from mvTestDup;")
createMV("create materialized view mv2 as select key2 from mvTestDup;")
createMV("create materialized view mv3 as select key1, key2, sum(value1), max(value2), min(value3) from mvTestDup group by key1, key2;")
@ -145,7 +134,7 @@ suite("test_analyze_mv") {
sql """analyze table mvTestDup with sync;"""
// Test show index row count
result_row = sql """show index stats mvTestDup mvTestDup"""
def result_row = sql """show index stats mvTestDup mvTestDup"""
assertEquals(1, result_row.size())
assertEquals("mvTestDup", result_row[0][0])
assertEquals("mvTestDup", result_row[0][1])
@ -474,35 +463,6 @@ suite("test_analyze_mv") {
logger.info(e.getMessage());
return;
}
if (!isCloudMode()) {
// Test row count report and report for nereids
result_row = sql """show index stats mvTestDup mvTestDup"""
assertEquals(1, result_row.size())
assertEquals("mvTestDup", result_row[0][0])
assertEquals("mvTestDup", result_row[0][1])
assertEquals("6", result_row[0][3])
assertEquals("6", result_row[0][4])
result_row = sql """show index stats mvTestDup mv1"""
assertEquals(1, result_row.size())
assertEquals("mvTestDup", result_row[0][0])
assertEquals("mv1", result_row[0][1])
assertEquals("6", result_row[0][3])
assertEquals("6", result_row[0][4])
result_row = sql """show index stats mvTestDup mv2"""
assertEquals(1, result_row.size())
assertEquals("mvTestDup", result_row[0][0])
assertEquals("mv2", result_row[0][1])
assertEquals("6", result_row[0][3])
assertEquals("6", result_row[0][4])
result_row = sql """show index stats mvTestDup mv3"""
assertEquals(1, result_row.size())
assertEquals("mvTestDup", result_row[0][0])
assertEquals("mv3", result_row[0][1])
assertEquals("4", result_row[0][3])
assertEquals("4", result_row[0][4])
}
sql """analyze table mvTestDup with sample rows 4000000"""
wait_analyze_finish("mvTestDup")
result_sample = sql """SHOW ANALYZE mvTestDup;"""
@ -650,38 +610,6 @@ suite("test_analyze_mv") {
verifyTaskStatus(result_sample, "mva_MIN__`value3`", "mv3")
verifyTaskStatus(result_sample, "mva_SUM__CAST(`value1` AS bigint)", "mv3")
if (!isCloudMode()) {
// Test row count report and report for nereids
sql """truncate table mvTestDup"""
result_row = sql """show index stats mvTestDup mv3"""
assertEquals(1, result_row.size())
assertEquals("mvTestDup", result_row[0][0])
assertEquals("mv3", result_row[0][1])
assertEquals("0", result_row[0][3])
assertEquals("-1", result_row[0][4])
for (int i = 0; i < 120; i++) {
result_row = sql """show index stats mvTestDup mv3"""
logger.info("mv3 stats: " + result_row)
if (result_row[0][4] == "0") {
break;
}
Thread.sleep(5000)
}
result_row = sql """show index stats mvTestDup mv3"""
assertEquals(1, result_row.size())
assertEquals("mvTestDup", result_row[0][0])
assertEquals("mv3", result_row[0][1])
assertEquals("0", result_row[0][3])
assertEquals("0", result_row[0][4])
sql """insert into mvTestDup values (1, 2, 3, 4, 5), (1, 2, 3, 4, 5), (10, 20, 30, 40, 50), (10, 20, 30, 40, 50), (100, 200, 300, 400, 500), (1001, 2001, 3001, 4001, 5001);"""
result_row = sql """show index stats mvTestDup mv3"""
assertEquals(1, result_row.size())
assertEquals("mvTestDup", result_row[0][0])
assertEquals("mv3", result_row[0][1])
assertEquals("-1", result_row[0][4])
}
// Test alter column stats
sql """drop stats mvTestDup"""
sql """alter table mvTestDup modify column key1 set stats ('ndv'='1', 'num_nulls'='1', 'min_value'='10', 'max_value'='40', 'row_count'='50');"""