database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[opt](statistics) Add wait row count reported logic for sample analyze. (#32030)

Browse Source 
If getRowCount returns 0, and table visible version is 2, the table is probably not empty, so we wait a moment for the row count to be reported.
This commit is contained in:
Jibing-Li
2024-03-13 10:51:39 +08:00
committed by yiguolei
parent c20567d088
commit 58675c271b
2 changed files with 27 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
fe/fe-core/src/main/java/org/apache/doris/statistics/OlapAnalysisTask.java
View File

@ -65,7 +65,7 @@ public class OlapAnalysisTask extends BaseAnalysisTask {
public void doExecute() throws Exception {
Set<String> partitionNames = info.colToPartitions.get(info.colName);
if ((info.emptyJob && info.analysisMethod.equals(AnalysisInfo.AnalysisMethod.SAMPLE))
if (StatisticsUtil.isEmptyTable(tbl, info.analysisMethod)
|| partitionNames == null || partitionNames.isEmpty()) {
if (partitionNames == null) {
LOG.warn("Table {}.{}.{}, partitionNames for column {} is null. ColToPartitions:[{}]",

26
fe/fe-core/src/main/java/org/apache/doris/statistics/util/StatisticsUtil.java
View File

@ -45,6 +45,7 @@ import org.apache.doris.catalog.PartitionItem;
import org.apache.doris.catalog.PrimitiveType;
import org.apache.doris.catalog.ScalarType;
import org.apache.doris.catalog.StructType;
import org.apache.doris.catalog.TableAttributes;
import org.apache.doris.catalog.TableIf;
import org.apache.doris.catalog.Type;
import org.apache.doris.catalog.VariantType;
@ -68,6 +69,7 @@ import org.apache.doris.qe.QueryState;
import org.apache.doris.qe.SessionVariable;
import org.apache.doris.qe.StmtExecutor;
import org.apache.doris.qe.VariableMgr;
import org.apache.doris.statistics.AnalysisInfo;
import org.apache.doris.statistics.ColumnStatistic;
import org.apache.doris.statistics.ColumnStatisticBuilder;
import org.apache.doris.statistics.Histogram;
@ -974,4 +976,28 @@ public class StatisticsUtil {
|| columnName.startsWith(CreateMaterializedViewStmt.MATERIALIZED_VIEW_AGGREGATE_NAME_PREFIX);
}
public static boolean isEmptyTable(TableIf table, AnalysisInfo.AnalysisMethod method) {
int waitRowCountReportedTime = 90;
if (!(table instanceof OlapTable) || method.equals(AnalysisInfo.AnalysisMethod.FULL)) {
return false;
}
OlapTable olapTable = (OlapTable) table;
for (int i = 0; i < waitRowCountReportedTime; i++) {
if (olapTable.getRowCount() > 0) {
return false;
}
// If visible version is 2, table is probably not empty. So we wait row count to be reported.
// If visible version is not 2 and getRowCount return 0, we assume it is an empty table.
if (olapTable.getVisibleVersion() != TableAttributes.TABLE_INIT_VERSION + 1) {
return true;
}
try {
Thread.sleep(1000);
} catch (InterruptedException e) {
LOG.info("Sleep interrupted.", e);
}
}
return true;
}
}