database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[fix](statistics)Fix empty table keep auto analyze bug. (#40811) (#40896)

Browse Source 
backport: https://github.com/apache/doris/pull/40811
This commit is contained in:
Jibing-Li
2024-09-18 12:51:44 +08:00
committed by GitHub
parent 4511d3e900
commit c176c9feb0
4 changed files with 73 additions and 21 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisInfoBuilder.java
View File

@ -63,7 +63,7 @@ public class AnalysisInfoBuilder {
private boolean usingSqlForPartitionColumn;
private long tblUpdateTime;
private boolean emptyJob;
private boolean userInject;
private boolean userInject = false;
private long rowCount;
public AnalysisInfoBuilder() {

16
fe/fe-core/src/main/java/org/apache/doris/statistics/TableStatsMeta.java
View File

@ -26,6 +26,7 @@ import org.apache.doris.common.io.Writable;
import org.apache.doris.persist.gson.GsonPostProcessable;
import org.apache.doris.persist.gson.GsonUtils;
import org.apache.doris.statistics.AnalysisInfo.JobType;
import org.apache.doris.statistics.util.StatisticsUtil;
import com.google.common.annotations.VisibleForTesting;
import com.google.gson.annotations.SerializedName;
@ -159,7 +160,9 @@ public class TableStatsMeta implements Writable, GsonPostProcessable {
public void update(AnalysisInfo analyzedJob, TableIf tableIf) {
updatedTime = analyzedJob.tblUpdateTime;
userInjected = analyzedJob.userInject;
if (analyzedJob.userInject) {
userInjected = true;
}
for (Pair<String, String> colPair : analyzedJob.jobColumns) {
ColStatsMeta colStatsMeta = colToColStatsMeta.get(colPair);
if (colStatsMeta == null) {
@ -179,15 +182,18 @@ public class TableStatsMeta implements Writable, GsonPostProcessable {
indexesRowCount.putAll(analyzedJob.indexesRowCount);
clearStaleIndexRowCount((OlapTable) tableIf);
}
if (analyzedJob.emptyJob) {
return;
}
if (analyzedJob.jobColumns.containsAll(
tableIf.getColumnIndexPairs(
tableIf.getSchemaAllIndexes(false).stream().map(Column::getName).collect(Collectors.toSet())))) {
tableIf.getSchemaAllIndexes(false).stream()
.filter(c -> !StatisticsUtil.isUnsupportedType(c.getType()))
.map(Column::getName).collect(Collectors.toSet())))) {
updatedRows.set(0);
newPartitionLoaded.set(false);
}
// Set userInject back to false after manual analyze.
if (JobType.MANUAL.equals(jobType) && !analyzedJob.userInject) {
userInjected = false;
}
}
}

28
fe/fe-core/src/main/java/org/apache/doris/statistics/util/StatisticsUtil.java
View File

@ -43,7 +43,6 @@ import org.apache.doris.catalog.Partition;
import org.apache.doris.catalog.PrimitiveType;
import org.apache.doris.catalog.ScalarType;
import org.apache.doris.catalog.StructType;
import org.apache.doris.catalog.TableAttributes;
import org.apache.doris.catalog.TableIf;
import org.apache.doris.catalog.Type;
import org.apache.doris.catalog.VariantType;
@ -878,27 +877,26 @@ public class StatisticsUtil {
}
public static boolean isEmptyTable(TableIf table, AnalysisInfo.AnalysisMethod method) {
int waitRowCountReportedTime = 90;
int waitRowCountReportedTime = 120;
if (!(table instanceof OlapTable) || method.equals(AnalysisInfo.AnalysisMethod.FULL)) {
return false;
}
OlapTable olapTable = (OlapTable) table;
long rowCount = 0;
for (int i = 0; i < waitRowCountReportedTime; i++) {
if (olapTable.getRowCount() > 0) {
return false;
}
// If visible version is 2, table is probably not empty. So we wait row count to be reported.
// If visible version is not 2 and getRowCount return 0, we assume it is an empty table.
if (olapTable.getVisibleVersion() != TableAttributes.TABLE_INIT_VERSION + 1) {
return true;
}
try {
Thread.sleep(1000);
} catch (InterruptedException e) {
LOG.info("Sleep interrupted.", e);
rowCount = olapTable.getRowCountForIndex(olapTable.getBaseIndexId(), true);
// rowCount == -1 means new table or first load row count not fully reported, need to wait.
if (rowCount == -1) {
try {
Thread.sleep(1000);
} catch (InterruptedException e) {
LOG.info("Sleep interrupted.");
}
continue;
}
break;
}
return true;
return rowCount == 0;
}
}

48
regression-test/suites/statistics/test_drop_stats_and_truncate.groovy
View File

@ -174,6 +174,54 @@ suite("test_drop_stats_and_truncate") {
columns = all_columns.split(",");
assertEquals(9, columns.size())
sql """drop table part"""
sql """CREATE TABLE `part` (
`id` INT NULL,
`colint` INT NULL,
`coltinyint` tinyint NULL,
`colsmallint` smallINT NULL,
`colbigint` bigINT NULL,
`collargeint` largeINT NULL,
`colfloat` float NULL,
`coldouble` double NULL,
`coldecimal` decimal(27, 9) NULL
) ENGINE=OLAP
DUPLICATE KEY(`id`)
COMMENT 'OLAP'
PARTITION BY RANGE(`id`)
(
PARTITION p1 VALUES [("-2147483648"), ("10000")),
PARTITION p2 VALUES [("10000"), ("20000")),
PARTITION p3 VALUES [("20000"), ("30000"))
)
DISTRIBUTED BY HASH(`id`) BUCKETS 3
PROPERTIES (
"replication_allocation" = "tag.location.default: 1"
)
"""
sql """analyze table part with sync"""
sql """Insert into part values (1, 1, 1, 1, 1, 1, 1.1, 1.1, 1.1)"""
result = sql """show table stats part"""
assertEquals("true", result[0][6])
sql """truncate table part partition(p1)"""
result = sql """show table stats part"""
assertEquals("true", result[0][6])
sql """analyze table part with sample rows 100 with sync"""
result = sql """show table stats part"""
if (result[0][6].equals("true")) {
result = """show index stats part part"""
logger.info("Report not ready. index stats: " + result)
sql """analyze table part with sample rows 100 with sync"""
result = sql """show table stats part"""
}
if (result[0][6].equals("true")) {
result = """show index stats part part"""
logger.info("Report not ready. index stats: " + result)
sql """analyze table part with sample rows 100 with sync"""
result = sql """show table stats part"""
}
assertEquals("false", result[0][6])
sql """drop database if exists test_drop_stats_and_truncate"""
}