database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[opt](Nereids)skip unknown col stats check on __internal_scheam and information_schema (#24625)

Browse Source 
columns in __internal_scheam and information_schema do not have column stats
This commit is contained in:
minghong
2023-09-20 10:48:05 +08:00
committed by GitHub
parent c41cadb64d
commit 9a4a4c0760
5 changed files with 32 additions and 22 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java
View File

@ -173,6 +173,7 @@ import org.apache.doris.planner.external.iceberg.IcebergScanNode;
import org.apache.doris.planner.external.jdbc.JdbcScanNode;
import org.apache.doris.planner.external.paimon.PaimonScanNode;
import org.apache.doris.qe.ConnectContext;
import org.apache.doris.statistics.StatisticConstants;
import org.apache.doris.tablefunction.TableValuedFunctionIf;
import org.apache.doris.thrift.TFetchOption;
import org.apache.doris.thrift.TPartitionType;
@ -581,7 +582,8 @@ public class PhysicalPlanTranslator extends DefaultPlanVisitor<PlanFragment, Pla
for (int i = 0; i < slots.size(); i++) {
Slot slot = slots.get(i);
if (olapScan.getStats().findColumnStatistics(slot).isUnKnown()
&& !isComplexDataType(slot.getDataType())) {
&& !isComplexDataType(slot.getDataType())
&& !StatisticConstants.isSystemTable(olapTable)) {
context.addUnknownStatsColumn(olapScanNode, tupleDescriptor.getSlots().get(i).getId());
}
}
@ -2043,7 +2045,8 @@ public class PhysicalPlanTranslator extends DefaultPlanVisitor<PlanFragment, Pla
scanNode.getTupleDesc().getSlots().add(smallest);
}
try {
if (ConnectContext.get() != null && ConnectContext.get().getSessionVariable().forbidUnknownColStats) {
if (ConnectContext.get() != null && ConnectContext.get().getSessionVariable().forbidUnknownColStats
&& !StatisticConstants.isSystemTable(scanNode.getTupleDesc().getTable())) {
for (SlotId slotId : requiredByProjectSlotIdSet) {
if (context.isColumnStatsUnknown(scanNode, slotId)) {
throw new AnalysisException("meet unknown column stats on table " + scanNode);

17
fe/fe-core/src/main/java/org/apache/doris/nereids/stats/StatsCalculator.java
View File

@ -18,11 +18,9 @@
package org.apache.doris.nereids.stats;
import org.apache.doris.analysis.IntLiteral;
import org.apache.doris.catalog.Column;
import org.apache.doris.catalog.Env;
import org.apache.doris.catalog.OlapTable;
import org.apache.doris.catalog.PartitionType;
import org.apache.doris.catalog.SchemaTable;
import org.apache.doris.catalog.TableIf;
import org.apache.doris.common.Config;
import org.apache.doris.common.FeConstants;
@ -627,7 +625,7 @@ public class StatsCalculator extends DefaultPlanVisitor<Statistics, Void> {
double rowCount = catalogRelation.getTable().estimatedRowCount();
for (SlotReference slotReference : slotSet) {
String colName = slotReference.getName();
boolean shouldIgnoreThisCol = shouldIgnoreCol(table, slotReference.getColumn().get());
boolean shouldIgnoreThisCol = StatisticConstants.shouldIgnoreCol(table, slotReference.getColumn().get());
if (colName == null) {
throw new RuntimeException(String.format("Invalid slot: %s", slotReference.getExprId()));
@ -1092,17 +1090,4 @@ public class StatsCalculator extends DefaultPlanVisitor<Statistics, Void> {
PhysicalCTEAnchor<? extends Plan, ? extends Plan> cteAnchor, Void context) {
return groupExpression.childStatistics(1);
}
private boolean shouldIgnoreCol(TableIf tableIf, Column c) {
if (tableIf instanceof SchemaTable) {
return true;
}
if (tableIf instanceof OlapTable) {
OlapTable olapTable = (OlapTable) tableIf;
if (StatisticConstants.STATISTICS_DB_BLACK_LIST.contains(olapTable.getQualifiedDbName())) {
return true;
}
}
return !c.isVisible();
}
}

26
fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticConstants.java
View File

@ -17,6 +17,9 @@
package org.apache.doris.statistics;
import org.apache.doris.catalog.Column;
import org.apache.doris.catalog.OlapTable;
import org.apache.doris.catalog.TableIf;
import org.apache.doris.cluster.ClusterNamespace;
import org.apache.doris.common.FeConstants;
import org.apache.doris.system.SystemInfoService;
@ -59,7 +62,7 @@ public class StatisticConstants {
public static final int ANALYZE_MANAGER_INTERVAL_IN_SECS = 60;
public static List<String> STATISTICS_DB_BLACK_LIST = new ArrayList<>();
public static List<String> SYSTEM_DBS = new ArrayList<>();
public static int ANALYZE_TASK_RETRY_TIMES = 5;
@ -79,9 +82,26 @@ public class StatisticConstants {
public static final int UNION_ALL_LIMIT = 512;
static {
STATISTICS_DB_BLACK_LIST.add(SystemInfoService.DEFAULT_CLUSTER
SYSTEM_DBS.add(SystemInfoService.DEFAULT_CLUSTER
+ ClusterNamespace.CLUSTER_DELIMITER + FeConstants.INTERNAL_DB_NAME);
STATISTICS_DB_BLACK_LIST.add(SystemInfoService.DEFAULT_CLUSTER
SYSTEM_DBS.add(SystemInfoService.DEFAULT_CLUSTER
+ ClusterNamespace.CLUSTER_DELIMITER + "information_schema");
}
public static boolean isSystemTable(TableIf tableIf) {
if (tableIf instanceof OlapTable) {
OlapTable olapTable = (OlapTable) tableIf;
if (StatisticConstants.SYSTEM_DBS.contains(olapTable.getQualifiedDbName())) {
return true;
}
}
return false;
}
public static boolean shouldIgnoreCol(TableIf tableIf, Column c) {
if (isSystemTable(tableIf)) {
return true;
}
return !c.isVisible();
}
}

2
fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsAutoCollector.java
View File

@ -76,7 +76,7 @@ public class StatisticsAutoCollector extends StatisticsCollector {
}
Collection<DatabaseIf> dbs = ctl.getAllDbs();
for (DatabaseIf<TableIf> databaseIf : dbs) {
if (StatisticConstants.STATISTICS_DB_BLACK_LIST.contains(databaseIf.getFullName())) {
if (StatisticConstants.SYSTEM_DBS.contains(databaseIf.getFullName())) {
continue;
}
analyzeDb(databaseIf);

2
regression-test/suites/nereids_p0/test_forbid_unknown_col_stats.groovy
View File

@ -49,5 +49,7 @@ suite("test_forbid_unknown_col_stats") {
exception "tables with unknown column stats: OlapScanNode{tid=0, tblName=test_forbid_unknown_col_stats_tbl, keyRanges=, preds= limit=-1}"
}
sql "select count() from __internal_schema.column_statistics"
sql "select count() from information_schema.views"
}