From 4e453dc1bb9769938a488ae78710fbfcd5b2aeee Mon Sep 17 00:00:00 2001 From: yiguolei <676222867@qq.com> Date: Tue, 10 Sep 2024 17:17:13 +0800 Subject: [PATCH] Revert "[improvement](statistics)Return -1 to neredis if report olap table row count for new table is not done for all tablets. (#40457)" (#40616) Reverts apache/doris#40540 --- be/src/olap/tablet_manager.cpp | 1 - .../doris/analysis/ShowTableStatsStmt.java | 53 ++++++------- .../doris/catalog/MaterializedIndex.java | 10 --- .../org/apache/doris/catalog/OlapTable.java | 13 ++-- .../org/apache/doris/catalog/Replica.java | 10 --- .../apache/doris/catalog/TabletStatMgr.java | 29 +------- .../org/apache/doris/qe/ShowExecutor.java | 11 ++- .../doris/statistics/AnalysisManager.java | 6 +- .../doris/statistics/OlapAnalysisTask.java | 2 +- .../doris/statistics/TableStatsMeta.java | 6 +- .../transaction/DatabaseTransactionMgr.java | 3 - gensrc/thrift/BackendService.thrift | 1 - .../suites/statistics/analyze_stats.groovy | 2 +- .../suites/statistics/test_analyze_mv.groovy | 74 +------------------ 14 files changed, 51 insertions(+), 170 deletions(-) diff --git a/be/src/olap/tablet_manager.cpp b/be/src/olap/tablet_manager.cpp index 05a91f463e..875876d3b1 100644 --- a/be/src/olap/tablet_manager.cpp +++ b/be/src/olap/tablet_manager.cpp @@ -1035,7 +1035,6 @@ void TabletManager::build_all_report_tablets_info(std::map* t_tablet_stat.__set_row_count(tablet_info.row_count); t_tablet_stat.__set_total_version_count(tablet_info.total_version_count); t_tablet_stat.__set_visible_version_count(tablet_info.visible_version_count); - t_tablet_stat.__set_visible_version(tablet_info.version); }; for_each_tablet(handler, filter_all_tablets); diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowTableStatsStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowTableStatsStmt.java index 91b8bf1de2..8a1feab285 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowTableStatsStmt.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowTableStatsStmt.java @@ -63,9 +63,7 @@ public class ShowTableStatsStmt extends ShowStmt { new ImmutableList.Builder() .add("table_name") .add("index_name") - .add("analyze_row_count") - .add("report_row_count") - .add("report_row_count_for_nereids") + .add("row_count") .build(); private final TableName tableName; @@ -169,33 +167,37 @@ public class ShowTableStatsStmt extends ShowStmt { return tableId; } - public ShowResultSet constructResultSet(TableStatsMeta tableStatistic, TableIf table) { + public ShowResultSet constructResultSet(TableStatsMeta tableStatistic) { if (indexName != null) { - return constructIndexResultSet(tableStatistic, table); + return constructIndexResultSet(tableStatistic); } - return constructTableResultSet(tableStatistic, table); + return constructTableResultSet(tableStatistic); } public ShowResultSet constructEmptyResultSet() { return new ShowResultSet(getMetaData(), new ArrayList<>()); } - public ShowResultSet constructTableResultSet(TableStatsMeta tableStatistic, TableIf table) { - if (tableStatistic == null) { - List> result = Lists.newArrayList(); - List row = Lists.newArrayList(); - row.add(""); - row.add(""); - row.add(String.valueOf(table.getCachedRowCount())); - row.add(""); - row.add(""); - row.add(""); - row.add(""); - row.add(""); - result.add(row); - return new ShowResultSet(getMetaData(), result); - } + public ShowResultSet constructResultSet(long rowCount) { + List> result = Lists.newArrayList(); + List row = Lists.newArrayList(); + row.add(""); + row.add(""); + row.add(String.valueOf(rowCount)); + row.add(""); + row.add(""); + row.add(""); + row.add(""); + row.add(""); + result.add(row); + return new ShowResultSet(getMetaData(), result); + } + + public ShowResultSet constructTableResultSet(TableStatsMeta tableStatistic) { DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss"); + if (tableStatistic == null) { + return new ShowResultSet(getMetaData(), new ArrayList<>()); + } List> result = Lists.newArrayList(); List row = Lists.newArrayList(); row.add(String.valueOf(tableStatistic.updatedRows)); @@ -214,7 +216,7 @@ public class ShowTableStatsStmt extends ShowStmt { return new ShowResultSet(getMetaData(), result); } - public ShowResultSet constructIndexResultSet(TableStatsMeta tableStatistic, TableIf table) { + public ShowResultSet constructIndexResultSet(TableStatsMeta tableStatistic) { List> result = Lists.newArrayList(); if (!(table instanceof OlapTable)) { return new ShowResultSet(getMetaData(), result); @@ -224,13 +226,14 @@ public class ShowTableStatsStmt extends ShowStmt { if (indexId == null) { throw new RuntimeException(String.format("Index %s not exist.", indexName)); } - long rowCount = tableStatistic == null ? -1 : tableStatistic.getRowCount(olapTable.getIndexIdByName(indexName)); + long rowCount = tableStatistic.getRowCount(olapTable.getIndexIdByName(indexName)); + if (rowCount == -1) { + return new ShowResultSet(getMetaData(), result); + } List row = Lists.newArrayList(); row.add(table.getName()); row.add(indexName); row.add(String.valueOf(rowCount)); - row.add(String.valueOf(olapTable.getRowCountForIndex(indexId, false))); - row.add(String.valueOf(olapTable.getRowCountForIndex(indexId, true))); result.add(row); return new ShowResultSet(getMetaData(), result); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/MaterializedIndex.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/MaterializedIndex.java index 4b70d96c28..094aebc6fc 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/MaterializedIndex.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/MaterializedIndex.java @@ -73,8 +73,6 @@ public class MaterializedIndex extends MetaObject implements Writable, GsonPostP @SerializedName(value = "rollupFinishedVersion") private long rollupFinishedVersion; - private boolean rowCountReported = false; - public MaterializedIndex() { this.state = IndexState.NORMAL; this.idToTablets = new HashMap<>(); @@ -208,14 +206,6 @@ public class MaterializedIndex extends MetaObject implements Writable, GsonPostP return -1; } - public void setRowCountReported(boolean reported) { - this.rowCountReported = reported; - } - - public boolean getRowCountReported() { - return this.rowCountReported; - } - @Override public void write(DataOutput out) throws IOException { super.write(out); diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java index 413dbdddcc..e048ae8ede 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java @@ -1396,17 +1396,18 @@ public class OlapTable extends Table implements MTMVRelatedTableIf { @Override public long fetchRowCount() { - return getRowCountForIndex(baseIndexId, false); + long rowCount = 0; + for (Map.Entry entry : idToPartition.entrySet()) { + rowCount += entry.getValue().getBaseIndex().getRowCount(); + } + return rowCount; } - public long getRowCountForIndex(long indexId, boolean strict) { + public long getRowCountForIndex(long indexId) { long rowCount = 0; for (Map.Entry entry : idToPartition.entrySet()) { MaterializedIndex index = entry.getValue().getIndex(indexId); - if (strict && !index.getRowCountReported()) { - return -1; - } - rowCount += (index == null || index.getRowCount() == -1) ? 0 : index.getRowCount(); + rowCount += index == null ? 0 : index.getRowCount(); } return rowCount; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Replica.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/Replica.java index c0e153f6ce..f6b2944d75 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Replica.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Replica.java @@ -164,8 +164,6 @@ public class Replica implements Writable { private long userDropTime = -1; - private long lastReportVersion = 0; - public Replica() { } @@ -813,12 +811,4 @@ public class Replica implements Writable { return Env.getCurrentSystemInfo().checkBackendScheduleAvailable(backendId) && !isUserDrop(); } - - public void setLastReportVersion(long version) { - this.lastReportVersion = version; - } - - public long getLastReportVersion() { - return lastReportVersion; - } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/TabletStatMgr.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/TabletStatMgr.java index fef57f37b5..896ecac6f8 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/TabletStatMgr.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/TabletStatMgr.java @@ -118,41 +118,17 @@ public class TabletStatMgr extends MasterDaemon { long version = partition.getVisibleVersion(); for (MaterializedIndex index : partition.getMaterializedIndices(IndexExtState.VISIBLE)) { long indexRowCount = 0L; - boolean indexReported = true; for (Tablet tablet : index.getTablets()) { long tabletRowCount = 0L; - boolean tabletReported = false; for (Replica replica : tablet.getReplicas()) { - LOG.debug("Table {} replica {} current version {}, report version {}", - olapTable.getName(), replica.getId(), - replica.getVersion(), replica.getLastReportVersion()); if (replica.checkVersionCatchUp(version, false) - && replica.getRowCount() >= tabletRowCount) { - // 1. If replica version and reported replica version are all equal to - // PARTITION_INIT_VERSION, set tabletReported to true, which indicates this - // tablet is empty for sure when previous report. - // 2. If last report version is larger than PARTITION_INIT_VERSION, set - // tabletReported to true as well. That is, we only guarantee all replicas of - // the tablet are reported for the init version. - // e.g. When replica version is 2, but last reported version is 1, - // tabletReported would be false. - if (replica.getVersion() == Partition.PARTITION_INIT_VERSION - && replica.getLastReportVersion() == Partition.PARTITION_INIT_VERSION - || replica.getLastReportVersion() > Partition.PARTITION_INIT_VERSION) { - tabletReported = true; - } + && replica.getRowCount() > tabletRowCount) { tabletRowCount = replica.getRowCount(); } } indexRowCount += tabletRowCount; - // Only when all tablets of this index are reported, we set indexReported to true. - indexReported = indexReported && tabletReported; } // end for tablets - index.setRowCountReported(indexReported); index.setRowCount(indexRowCount); - LOG.debug("Table {} index {} all tablets reported[{}], row count {}", - olapTable.getName(), olapTable.getIndexNameById(index.getId()), - indexReported, indexRowCount); } // end for indices } // end for partitions if (LOG.isDebugEnabled()) { @@ -181,9 +157,6 @@ public class TabletStatMgr extends MasterDaemon { replica.setTotalVersionCount(stat.getTotalVersionCount()); replica.setVisibleVersionCount(stat.isSetVisibleVersionCount() ? stat.getVisibleVersionCount() : stat.getTotalVersionCount()); - // Older version BE doesn't set visible version. Set it to max for compatibility. - replica.setLastReportVersion(stat.isSetVisibleVersion() ? stat.getVisibleVersion() - : Long.MAX_VALUE); } } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java b/fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java index 94878f0533..7652e8bb2b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java @@ -2520,12 +2520,19 @@ public class ShowExecutor { if (tableStats == null) { resultSet = showTableStatsStmt.constructEmptyResultSet(); } else { - resultSet = showTableStatsStmt.constructResultSet(tableStats, tableIf); + resultSet = showTableStatsStmt.constructResultSet(tableStats); } return; } TableStatsMeta tableStats = Env.getCurrentEnv().getAnalysisManager().findTableStatsStatus(tableIf.getId()); - resultSet = showTableStatsStmt.constructResultSet(tableStats, tableIf); + /* + tableStats == null means it's not analyzed, in this case show the estimated row count. + */ + if (tableStats == null) { + resultSet = showTableStatsStmt.constructResultSet(tableIf.getCachedRowCount()); + } else { + resultSet = showTableStatsStmt.constructResultSet(tableStats); + } } private void handleShowColumnStats() throws AnalysisException { diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisManager.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisManager.java index 6121dfd83e..204b60cfd0 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisManager.java +++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisManager.java @@ -640,7 +640,7 @@ public class AnalysisManager implements Writable { if (tableStats == null) { return; } - invalidateLocalStats(catalogId, dbId, tblId, dropStatsStmt.isAllColumns() ? null : cols, tableStats); + invalidateLocalStats(catalogId, dbId, tblId, cols, tableStats); // Drop stats ddl is master only operation. invalidateRemoteStats(catalogId, dbId, tblId, cols, dropStatsStmt.isAllColumns()); StatisticsRepository.dropStatisticsByColNames(catalogId, dbId, tblId, cols); @@ -655,7 +655,7 @@ public class AnalysisManager implements Writable { long dbId = table.getDatabase().getId(); long tableId = table.getId(); Set cols = table.getSchemaAllIndexes(false).stream().map(Column::getName).collect(Collectors.toSet()); - invalidateLocalStats(catalogId, dbId, tableId, null, tableStats); + invalidateLocalStats(catalogId, dbId, tableId, cols, tableStats); // Drop stats ddl is master only operation. invalidateRemoteStats(catalogId, dbId, tableId, cols, true); StatisticsRepository.dropStatisticsByColNames(catalogId, dbId, table.getId(), cols); @@ -717,8 +717,6 @@ public class AnalysisManager implements Writable { // To remove stale column name that is changed before. if (allColumn) { tableStats.removeAllColumn(); - tableStats.clearIndexesRowCount(); - removeTableStats(tableId); } tableStats.updatedTime = 0; tableStats.userInjected = false; diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/OlapAnalysisTask.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/OlapAnalysisTask.java index 7f96c52e81..e9bc9afbe2 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/statistics/OlapAnalysisTask.java +++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/OlapAnalysisTask.java @@ -102,7 +102,7 @@ public class OlapAnalysisTask extends BaseAnalysisTask { List tabletIds = pair.first; long totalRowCount = info.indexId == -1 ? tbl.getRowCount() - : ((OlapTable) tbl).getRowCountForIndex(info.indexId, false); + : ((OlapTable) tbl).getRowCountForIndex(info.indexId); double scaleFactor = (double) totalRowCount / (double) pair.second; // might happen if row count in fe metadata hasn't been updated yet if (Double.isInfinite(scaleFactor) || Double.isNaN(scaleFactor)) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/TableStatsMeta.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/TableStatsMeta.java index f2af3e8827..aa3d1e6a4e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/statistics/TableStatsMeta.java +++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/TableStatsMeta.java @@ -95,7 +95,7 @@ public class TableStatsMeta implements Writable, GsonPostProcessable { public boolean userInjected; @SerializedName("irc") - private ConcurrentMap indexesRowCount = new ConcurrentHashMap<>(); + public ConcurrentMap indexesRowCount = new ConcurrentHashMap<>(); @VisibleForTesting public TableStatsMeta() { @@ -212,10 +212,6 @@ public class TableStatsMeta implements Writable, GsonPostProcessable { return indexesRowCount.getOrDefault(indexId, -1L); } - public void clearIndexesRowCount() { - indexesRowCount.clear(); - } - private void clearStaleIndexRowCount(OlapTable table) { Iterator iterator = indexesRowCount.keySet().iterator(); List indexIds = table.getIndexIds(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/transaction/DatabaseTransactionMgr.java b/fe/fe-core/src/main/java/org/apache/doris/transaction/DatabaseTransactionMgr.java index e779476975..38514aff8d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/transaction/DatabaseTransactionMgr.java +++ b/fe/fe-core/src/main/java/org/apache/doris/transaction/DatabaseTransactionMgr.java @@ -2102,9 +2102,6 @@ public class DatabaseTransactionMgr { } } replica.updateVersionWithFailed(newVersion, lastFailedVersion, lastSuccessVersion); - if (newVersion == Partition.PARTITION_INIT_VERSION + 1) { - index.setRowCountReported(false); - } Set partitionIds = backendPartitions.get(replica.getBackendId()); if (partitionIds == null) { partitionIds = Sets.newHashSet(); diff --git a/gensrc/thrift/BackendService.thrift b/gensrc/thrift/BackendService.thrift index 6a5e403506..4714b38b4c 100644 --- a/gensrc/thrift/BackendService.thrift +++ b/gensrc/thrift/BackendService.thrift @@ -37,7 +37,6 @@ struct TTabletStat { 4: optional i64 total_version_count 5: optional i64 remote_data_size 6: optional i64 visible_version_count - 7: optional i64 visible_version } struct TTabletStatResult { diff --git a/regression-test/suites/statistics/analyze_stats.groovy b/regression-test/suites/statistics/analyze_stats.groovy index 50ba18e97b..09a8ecbcad 100644 --- a/regression-test/suites/statistics/analyze_stats.groovy +++ b/regression-test/suites/statistics/analyze_stats.groovy @@ -2753,7 +2753,7 @@ PARTITION `p599` VALUES IN (599) assertEquals("true", alter_result[0][7]) sql """drop stats alter_test""" alter_result = sql """show table stats alter_test""" - assertEquals("", alter_result[0][7]) + assertEquals("false", alter_result[0][7]) sql """alter table alter_test modify column id set stats ('row_count'='100', 'ndv'='0', 'num_nulls'='0.0', 'data_size'='2.69975443E8', 'min_value'='1', 'max_value'='2');""" alter_result = sql """show column stats alter_test(id)""" logger.info("show column alter_test(id) stats: " + alter_result) diff --git a/regression-test/suites/statistics/test_analyze_mv.groovy b/regression-test/suites/statistics/test_analyze_mv.groovy index add93f2a95..d5acedf614 100644 --- a/regression-test/suites/statistics/test_analyze_mv.groovy +++ b/regression-test/suites/statistics/test_analyze_mv.groovy @@ -126,17 +126,6 @@ suite("test_analyze_mv") { "replication_num" = "1" ) """ - def result_row - if (!isCloudMode()) { - // Test row count report and report for nereids - result_row = sql """show index stats mvTestDup mvTestDup""" - assertEquals(1, result_row.size()) - assertEquals("mvTestDup", result_row[0][0]) - assertEquals("mvTestDup", result_row[0][1]) - assertEquals("0", result_row[0][3]) - assertEquals("-1", result_row[0][4]) - } - createMV("create materialized view mv1 as select key1 from mvTestDup;") createMV("create materialized view mv2 as select key2 from mvTestDup;") createMV("create materialized view mv3 as select key1, key2, sum(value1), max(value2), min(value3) from mvTestDup group by key1, key2;") @@ -145,7 +134,7 @@ suite("test_analyze_mv") { sql """analyze table mvTestDup with sync;""" // Test show index row count - result_row = sql """show index stats mvTestDup mvTestDup""" + def result_row = sql """show index stats mvTestDup mvTestDup""" assertEquals(1, result_row.size()) assertEquals("mvTestDup", result_row[0][0]) assertEquals("mvTestDup", result_row[0][1]) @@ -474,35 +463,6 @@ suite("test_analyze_mv") { logger.info(e.getMessage()); return; } - - if (!isCloudMode()) { - // Test row count report and report for nereids - result_row = sql """show index stats mvTestDup mvTestDup""" - assertEquals(1, result_row.size()) - assertEquals("mvTestDup", result_row[0][0]) - assertEquals("mvTestDup", result_row[0][1]) - assertEquals("6", result_row[0][3]) - assertEquals("6", result_row[0][4]) - result_row = sql """show index stats mvTestDup mv1""" - assertEquals(1, result_row.size()) - assertEquals("mvTestDup", result_row[0][0]) - assertEquals("mv1", result_row[0][1]) - assertEquals("6", result_row[0][3]) - assertEquals("6", result_row[0][4]) - result_row = sql """show index stats mvTestDup mv2""" - assertEquals(1, result_row.size()) - assertEquals("mvTestDup", result_row[0][0]) - assertEquals("mv2", result_row[0][1]) - assertEquals("6", result_row[0][3]) - assertEquals("6", result_row[0][4]) - result_row = sql """show index stats mvTestDup mv3""" - assertEquals(1, result_row.size()) - assertEquals("mvTestDup", result_row[0][0]) - assertEquals("mv3", result_row[0][1]) - assertEquals("4", result_row[0][3]) - assertEquals("4", result_row[0][4]) - } - sql """analyze table mvTestDup with sample rows 4000000""" wait_analyze_finish("mvTestDup") result_sample = sql """SHOW ANALYZE mvTestDup;""" @@ -650,38 +610,6 @@ suite("test_analyze_mv") { verifyTaskStatus(result_sample, "mva_MIN__`value3`", "mv3") verifyTaskStatus(result_sample, "mva_SUM__CAST(`value1` AS bigint)", "mv3") - if (!isCloudMode()) { - // Test row count report and report for nereids - sql """truncate table mvTestDup""" - result_row = sql """show index stats mvTestDup mv3""" - assertEquals(1, result_row.size()) - assertEquals("mvTestDup", result_row[0][0]) - assertEquals("mv3", result_row[0][1]) - assertEquals("0", result_row[0][3]) - assertEquals("-1", result_row[0][4]) - - for (int i = 0; i < 120; i++) { - result_row = sql """show index stats mvTestDup mv3""" - logger.info("mv3 stats: " + result_row) - if (result_row[0][4] == "0") { - break; - } - Thread.sleep(5000) - } - result_row = sql """show index stats mvTestDup mv3""" - assertEquals(1, result_row.size()) - assertEquals("mvTestDup", result_row[0][0]) - assertEquals("mv3", result_row[0][1]) - assertEquals("0", result_row[0][3]) - assertEquals("0", result_row[0][4]) - sql """insert into mvTestDup values (1, 2, 3, 4, 5), (1, 2, 3, 4, 5), (10, 20, 30, 40, 50), (10, 20, 30, 40, 50), (100, 200, 300, 400, 500), (1001, 2001, 3001, 4001, 5001);""" - result_row = sql """show index stats mvTestDup mv3""" - assertEquals(1, result_row.size()) - assertEquals("mvTestDup", result_row[0][0]) - assertEquals("mv3", result_row[0][1]) - assertEquals("-1", result_row[0][4]) - } - // Test alter column stats sql """drop stats mvTestDup""" sql """alter table mvTestDup modify column key1 set stats ('ndv'='1', 'num_nulls'='1', 'min_value'='10', 'max_value'='40', 'row_count'='50');"""