From 9a4a4c0760cfd15fa4e45dbf140f326f8644b826 Mon Sep 17 00:00:00 2001 From: minghong Date: Wed, 20 Sep 2023 10:48:05 +0800 Subject: [PATCH] [opt](Nereids)skip unknown col stats check on __internal_scheam and information_schema (#24625) columns in __internal_scheam and information_schema do not have column stats --- .../translator/PhysicalPlanTranslator.java | 7 +++-- .../doris/nereids/stats/StatsCalculator.java | 17 +----------- .../doris/statistics/StatisticConstants.java | 26 ++++++++++++++++--- .../statistics/StatisticsAutoCollector.java | 2 +- .../test_forbid_unknown_col_stats.groovy | 2 ++ 5 files changed, 32 insertions(+), 22 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java index 59f4318811..5db8d54d2b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java @@ -173,6 +173,7 @@ import org.apache.doris.planner.external.iceberg.IcebergScanNode; import org.apache.doris.planner.external.jdbc.JdbcScanNode; import org.apache.doris.planner.external.paimon.PaimonScanNode; import org.apache.doris.qe.ConnectContext; +import org.apache.doris.statistics.StatisticConstants; import org.apache.doris.tablefunction.TableValuedFunctionIf; import org.apache.doris.thrift.TFetchOption; import org.apache.doris.thrift.TPartitionType; @@ -581,7 +582,8 @@ public class PhysicalPlanTranslator extends DefaultPlanVisitor { double rowCount = catalogRelation.getTable().estimatedRowCount(); for (SlotReference slotReference : slotSet) { String colName = slotReference.getName(); - boolean shouldIgnoreThisCol = shouldIgnoreCol(table, slotReference.getColumn().get()); + boolean shouldIgnoreThisCol = StatisticConstants.shouldIgnoreCol(table, slotReference.getColumn().get()); if (colName == null) { throw new RuntimeException(String.format("Invalid slot: %s", slotReference.getExprId())); @@ -1092,17 +1090,4 @@ public class StatsCalculator extends DefaultPlanVisitor { PhysicalCTEAnchor cteAnchor, Void context) { return groupExpression.childStatistics(1); } - - private boolean shouldIgnoreCol(TableIf tableIf, Column c) { - if (tableIf instanceof SchemaTable) { - return true; - } - if (tableIf instanceof OlapTable) { - OlapTable olapTable = (OlapTable) tableIf; - if (StatisticConstants.STATISTICS_DB_BLACK_LIST.contains(olapTable.getQualifiedDbName())) { - return true; - } - } - return !c.isVisible(); - } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticConstants.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticConstants.java index c77ffa49ed..4bb7030753 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticConstants.java +++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticConstants.java @@ -17,6 +17,9 @@ package org.apache.doris.statistics; +import org.apache.doris.catalog.Column; +import org.apache.doris.catalog.OlapTable; +import org.apache.doris.catalog.TableIf; import org.apache.doris.cluster.ClusterNamespace; import org.apache.doris.common.FeConstants; import org.apache.doris.system.SystemInfoService; @@ -59,7 +62,7 @@ public class StatisticConstants { public static final int ANALYZE_MANAGER_INTERVAL_IN_SECS = 60; - public static List STATISTICS_DB_BLACK_LIST = new ArrayList<>(); + public static List SYSTEM_DBS = new ArrayList<>(); public static int ANALYZE_TASK_RETRY_TIMES = 5; @@ -79,9 +82,26 @@ public class StatisticConstants { public static final int UNION_ALL_LIMIT = 512; static { - STATISTICS_DB_BLACK_LIST.add(SystemInfoService.DEFAULT_CLUSTER + SYSTEM_DBS.add(SystemInfoService.DEFAULT_CLUSTER + ClusterNamespace.CLUSTER_DELIMITER + FeConstants.INTERNAL_DB_NAME); - STATISTICS_DB_BLACK_LIST.add(SystemInfoService.DEFAULT_CLUSTER + SYSTEM_DBS.add(SystemInfoService.DEFAULT_CLUSTER + ClusterNamespace.CLUSTER_DELIMITER + "information_schema"); } + + public static boolean isSystemTable(TableIf tableIf) { + if (tableIf instanceof OlapTable) { + OlapTable olapTable = (OlapTable) tableIf; + if (StatisticConstants.SYSTEM_DBS.contains(olapTable.getQualifiedDbName())) { + return true; + } + } + return false; + } + + public static boolean shouldIgnoreCol(TableIf tableIf, Column c) { + if (isSystemTable(tableIf)) { + return true; + } + return !c.isVisible(); + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsAutoCollector.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsAutoCollector.java index 64ea89ff63..3b8b7840aa 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsAutoCollector.java +++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsAutoCollector.java @@ -76,7 +76,7 @@ public class StatisticsAutoCollector extends StatisticsCollector { } Collection dbs = ctl.getAllDbs(); for (DatabaseIf databaseIf : dbs) { - if (StatisticConstants.STATISTICS_DB_BLACK_LIST.contains(databaseIf.getFullName())) { + if (StatisticConstants.SYSTEM_DBS.contains(databaseIf.getFullName())) { continue; } analyzeDb(databaseIf); diff --git a/regression-test/suites/nereids_p0/test_forbid_unknown_col_stats.groovy b/regression-test/suites/nereids_p0/test_forbid_unknown_col_stats.groovy index a0d9d75499..7b4ae75bfa 100644 --- a/regression-test/suites/nereids_p0/test_forbid_unknown_col_stats.groovy +++ b/regression-test/suites/nereids_p0/test_forbid_unknown_col_stats.groovy @@ -49,5 +49,7 @@ suite("test_forbid_unknown_col_stats") { exception "tables with unknown column stats: OlapScanNode{tid=0, tblName=test_forbid_unknown_col_stats_tbl, keyRanges=, preds= limit=-1}" } + sql "select count() from __internal_schema.column_statistics" + sql "select count() from information_schema.views" }