From 86d7a8be4442bdb74d79f1ce6043e72e4093fa90 Mon Sep 17 00:00:00 2001 From: Jibing-Li <64681310+Jibing-Li@users.noreply.github.com> Date: Wed, 24 Jan 2024 19:27:15 +0800 Subject: [PATCH] [improvement](statistics nereids)Nereids support select mv. (#30267) --- .../org/apache/doris/nereids/DorisParser.g4 | 6 +- .../java/org/apache/doris/load/ExportJob.java | 2 +- .../nereids/analyzer/UnboundRelation.java | 20 ++-- .../nereids/parser/LogicalPlanBuilder.java | 7 +- .../nereids/rules/analysis/BindRelation.java | 19 +++- .../implementation/AggregateStrategies.java | 6 +- .../trees/plans/logical/LogicalOlapScan.java | 35 +++++-- .../data/statistics/test_select_mv.out | 47 +++++++++ .../suites/statistics/test_select_mv.groovy | 97 +++++++++++++++++++ 9 files changed, 215 insertions(+), 24 deletions(-) create mode 100644 regression-test/data/statistics/test_select_mv.out create mode 100644 regression-test/suites/statistics/test_select_mv.groovy diff --git a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 index 4ec4065ce7..2afcc4d524 100644 --- a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 +++ b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 @@ -439,7 +439,7 @@ identifierSeq ; relationPrimary - : multipartIdentifier specifiedPartition? + : multipartIdentifier materializedViewName? specifiedPartition? tabletList? tableAlias sample? relationHint? lateralView* #tableName | LEFT_PAREN query RIGHT_PAREN tableAlias lateralView* #aliasedQuery | tvfName=identifier LEFT_PAREN @@ -447,6 +447,10 @@ relationPrimary RIGHT_PAREN tableAlias #tableValuedFunction ; +materializedViewName + : INDEX indexName=identifier + ; + propertyClause : PROPERTIES LEFT_PAREN fileProperties=propertyItemList RIGHT_PAREN ; diff --git a/fe/fe-core/src/main/java/org/apache/doris/load/ExportJob.java b/fe/fe-core/src/main/java/org/apache/doris/load/ExportJob.java index fc2f6fca9c..0ae513d1db 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/load/ExportJob.java +++ b/fe/fe-core/src/main/java/org/apache/doris/load/ExportJob.java @@ -353,7 +353,7 @@ public class ExportJob implements Writable { List partitions, List selectLists) { // UnboundRelation LogicalPlan plan = new UnboundRelation(StatementScopeIdGenerator.newRelationId(), qualifiedTableName, - partitions, false, tabletIds, ImmutableList.of(), Optional.empty()); + partitions, false, tabletIds, ImmutableList.of(), Optional.empty(), Optional.empty()); // LogicalCheckPolicy plan = new LogicalCheckPolicy<>(plan); // LogicalFilter diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundRelation.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundRelation.java index 080e749c05..74f85e3165 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundRelation.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundRelation.java @@ -52,21 +52,22 @@ public class UnboundRelation extends LogicalRelation implements Unbound, BlockFu private final boolean isTempPart; private final List hints; private final Optional tableSample; + private final Optional indexName; public UnboundRelation(RelationId id, List nameParts) { this(id, nameParts, Optional.empty(), Optional.empty(), ImmutableList.of(), false, ImmutableList.of(), - ImmutableList.of(), Optional.empty()); + ImmutableList.of(), Optional.empty(), Optional.empty()); } public UnboundRelation(RelationId id, List nameParts, List partNames, boolean isTempPart) { this(id, nameParts, Optional.empty(), Optional.empty(), partNames, isTempPart, ImmutableList.of(), - ImmutableList.of(), Optional.empty()); + ImmutableList.of(), Optional.empty(), Optional.empty()); } public UnboundRelation(RelationId id, List nameParts, List partNames, boolean isTempPart, - List tabletIds, List hints, Optional tableSample) { + List tabletIds, List hints, Optional tableSample, Optional indexName) { this(id, nameParts, Optional.empty(), Optional.empty(), - partNames, isTempPart, tabletIds, hints, tableSample); + partNames, isTempPart, tabletIds, hints, tableSample, indexName); } /** @@ -74,7 +75,7 @@ public class UnboundRelation extends LogicalRelation implements Unbound, BlockFu */ public UnboundRelation(RelationId id, List nameParts, Optional groupExpression, Optional logicalProperties, List partNames, boolean isTempPart, - List tabletIds, List hints, Optional tableSample) { + List tabletIds, List hints, Optional tableSample, Optional indexName) { super(id, PlanType.LOGICAL_UNBOUND_RELATION, groupExpression, logicalProperties); this.nameParts = ImmutableList.copyOf(Objects.requireNonNull(nameParts, "nameParts should not null")); this.partNames = ImmutableList.copyOf(Objects.requireNonNull(partNames, "partNames should not null")); @@ -82,6 +83,7 @@ public class UnboundRelation extends LogicalRelation implements Unbound, BlockFu this.isTempPart = isTempPart; this.hints = ImmutableList.copyOf(Objects.requireNonNull(hints, "hints should not be null.")); this.tableSample = tableSample; + this.indexName = indexName; } public List getNameParts() { @@ -102,14 +104,14 @@ public class UnboundRelation extends LogicalRelation implements Unbound, BlockFu public Plan withGroupExpression(Optional groupExpression) { return new UnboundRelation(relationId, nameParts, groupExpression, Optional.of(getLogicalProperties()), - partNames, isTempPart, tabletIds, hints, tableSample); + partNames, isTempPart, tabletIds, hints, tableSample, indexName); } @Override public Plan withGroupExprLogicalPropChildren(Optional groupExpression, Optional logicalProperties, List children) { return new UnboundRelation(relationId, nameParts, groupExpression, logicalProperties, partNames, - isTempPart, tabletIds, hints, tableSample); + isTempPart, tabletIds, hints, tableSample, indexName); } @Override @@ -152,6 +154,10 @@ public class UnboundRelation extends LogicalRelation implements Unbound, BlockFu return tabletIds; } + public Optional getIndexName() { + return indexName; + } + public List getHints() { return hints; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java index 9bafca42f9..6da5a04fe1 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java @@ -1248,6 +1248,11 @@ public class LogicalPlanBuilder extends DorisParserBaseVisitor { } } + Optional indexName = Optional.empty(); + if (ctx.materializedViewName() != null) { + indexName = Optional.ofNullable(ctx.materializedViewName().indexName.getText()); + } + List tabletIdLists = new ArrayList<>(); if (ctx.tabletList() != null) { ctx.tabletList().tabletIdList.stream().forEach(tabletToken -> { @@ -1266,7 +1271,7 @@ public class LogicalPlanBuilder extends DorisParserBaseVisitor { LogicalPlan checkedRelation = LogicalPlanBuilderAssistant.withCheckPolicy( new UnboundRelation(StatementScopeIdGenerator.newRelationId(), tableId, partitionNames, isTempPart, tabletIdLists, relationHints, - Optional.ofNullable(tableSample))); + Optional.ofNullable(tableSample), indexName)); LogicalPlan plan = withTableAlias(checkedRelation, ctx.tableAlias()); for (LateralViewContext lateralViewContext : ctx.lateralView()) { plan = withGenerate(plan, lateralViewContext); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindRelation.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindRelation.java index f97fa4eeb4..c955aa5b17 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindRelation.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindRelation.java @@ -189,13 +189,26 @@ public class BindRelation extends OneAnalysisRuleFactory { (OlapTable) table, ImmutableList.of(tableQualifier.get(1)), partIds, tabletIds, unboundRelation.getHints(), unboundRelation.getTableSample()); } else { - scan = new LogicalOlapScan(unboundRelation.getRelationId(), + Optional indexName = unboundRelation.getIndexName(); + if (indexName.isPresent()) { + OlapTable olapTable = (OlapTable) table; + Long indexId = olapTable.getIndexIdByName(indexName.get()); + if (indexId == null) { + throw new AnalysisException("Table " + olapTable.getName() + + " doesn't have materialized view " + indexName.get()); + } + scan = new LogicalOlapScan(unboundRelation.getRelationId(), + (OlapTable) table, ImmutableList.of(tableQualifier.get(1)), tabletIds, indexId, + unboundRelation.getHints(), unboundRelation.getTableSample()); + } else { + scan = new LogicalOlapScan(unboundRelation.getRelationId(), (OlapTable) table, ImmutableList.of(tableQualifier.get(1)), tabletIds, unboundRelation.getHints(), unboundRelation.getTableSample()); + } } if (!Util.showHiddenColumns() && scan.getTable().hasDeleteSign() - && !ConnectContext.get().getSessionVariable() - .skipDeleteSign()) { + && !ConnectContext.get().getSessionVariable().skipDeleteSign() + && !scan.isDirectMvScan()) { // table qualifier is catalog.db.table, we make db.table.column Slot deleteSlot = null; for (Slot slot : scan.getOutput()) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/AggregateStrategies.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/AggregateStrategies.java index c9907ae7c3..254e014240 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/AggregateStrategies.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/AggregateStrategies.java @@ -515,10 +515,14 @@ public class AggregateStrategies implements ImplementationRuleFactory { return canNotPush; } if (logicalScan instanceof LogicalOlapScan) { - KeysType keysType = ((LogicalOlapScan) logicalScan).getTable().getKeysType(); + LogicalOlapScan logicalOlapScan = (LogicalOlapScan) logicalScan; + KeysType keysType = logicalOlapScan.getTable().getKeysType(); if (functionClasses.contains(Count.class) && keysType != KeysType.DUP_KEYS) { return canNotPush; } + if (functionClasses.contains(Count.class) && logicalOlapScan.isDirectMvScan()) { + return canNotPush; + } } if (aggregateFunctions.stream().anyMatch(fun -> fun.arity() > 1)) { return canNotPush; diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalOlapScan.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalOlapScan.java index 6916631898..f496eaa47f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalOlapScan.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalOlapScan.java @@ -108,6 +108,8 @@ public class LogicalOlapScan extends LogicalCatalogRelation implements OlapScan private final Optional tableSample; + private final boolean directMvScan; + public LogicalOlapScan(RelationId id, OlapTable table) { this(id, table, ImmutableList.of()); } @@ -117,7 +119,7 @@ public class LogicalOlapScan extends LogicalCatalogRelation implements OlapScan table.getPartitionIds(), false, ImmutableList.of(), -1, false, PreAggStatus.on(), ImmutableList.of(), ImmutableList.of(), - Maps.newHashMap(), Optional.empty()); + Maps.newHashMap(), Optional.empty(), false); } public LogicalOlapScan(RelationId id, OlapTable table, List qualifier, List tabletIds, @@ -125,7 +127,7 @@ public class LogicalOlapScan extends LogicalCatalogRelation implements OlapScan this(id, table, qualifier, Optional.empty(), Optional.empty(), table.getPartitionIds(), false, tabletIds, -1, false, PreAggStatus.on(), ImmutableList.of(), hints, Maps.newHashMap(), - tableSample); + tableSample, false); } public LogicalOlapScan(RelationId id, OlapTable table, List qualifier, List specifiedPartitions, @@ -134,7 +136,15 @@ public class LogicalOlapScan extends LogicalCatalogRelation implements OlapScan // must use specifiedPartitions here for prune partition by sql like 'select * from t partition p1' specifiedPartitions, false, tabletIds, -1, false, PreAggStatus.on(), specifiedPartitions, hints, Maps.newHashMap(), - tableSample); + tableSample, false); + } + + public LogicalOlapScan(RelationId id, OlapTable table, List qualifier, List tabletIds, + long selectedIndexId, List hints, Optional tableSample) { + this(id, table, qualifier, Optional.empty(), Optional.empty(), + table.getPartitionIds(), false, tabletIds, + selectedIndexId, true, PreAggStatus.off("For direct index scan."), + ImmutableList.of(), hints, Maps.newHashMap(), tableSample, true); } /** @@ -146,7 +156,7 @@ public class LogicalOlapScan extends LogicalCatalogRelation implements OlapScan List selectedTabletIds, long selectedIndexId, boolean indexSelected, PreAggStatus preAggStatus, List specifiedPartitions, List hints, Map, Slot> cacheSlotWithSlotName, - Optional tableSample) { + Optional tableSample, boolean directMvScan) { super(id, PlanType.LOGICAL_OLAP_SCAN, table, qualifier, groupExpression, logicalProperties); Preconditions.checkArgument(selectedPartitionIds != null, @@ -164,6 +174,7 @@ public class LogicalOlapScan extends LogicalCatalogRelation implements OlapScan this.cacheSlotWithSlotName = Objects.requireNonNull(cacheSlotWithSlotName, "mvNameToSlot can not be null"); this.tableSample = tableSample; + this.directMvScan = directMvScan; } public List getSelectedPartitionIds() { @@ -220,7 +231,7 @@ public class LogicalOlapScan extends LogicalCatalogRelation implements OlapScan groupExpression, Optional.of(getLogicalProperties()), selectedPartitionIds, partitionPruned, selectedTabletIds, selectedIndexId, indexSelected, preAggStatus, manuallySpecifiedPartitions, - hints, cacheSlotWithSlotName, tableSample); + hints, cacheSlotWithSlotName, tableSample, directMvScan); } @Override @@ -229,7 +240,7 @@ public class LogicalOlapScan extends LogicalCatalogRelation implements OlapScan return new LogicalOlapScan(relationId, (Table) table, qualifier, groupExpression, logicalProperties, selectedPartitionIds, partitionPruned, selectedTabletIds, selectedIndexId, indexSelected, preAggStatus, manuallySpecifiedPartitions, - hints, cacheSlotWithSlotName, tableSample); + hints, cacheSlotWithSlotName, tableSample, directMvScan); } public LogicalOlapScan withSelectedPartitionIds(List selectedPartitionIds) { @@ -237,7 +248,7 @@ public class LogicalOlapScan extends LogicalCatalogRelation implements OlapScan Optional.empty(), Optional.of(getLogicalProperties()), selectedPartitionIds, true, selectedTabletIds, selectedIndexId, indexSelected, preAggStatus, manuallySpecifiedPartitions, - hints, cacheSlotWithSlotName, tableSample); + hints, cacheSlotWithSlotName, tableSample, directMvScan); } public LogicalOlapScan withMaterializedIndexSelected(PreAggStatus preAgg, long indexId) { @@ -245,7 +256,7 @@ public class LogicalOlapScan extends LogicalCatalogRelation implements OlapScan Optional.empty(), Optional.of(getLogicalProperties()), selectedPartitionIds, partitionPruned, selectedTabletIds, indexId, true, preAgg, manuallySpecifiedPartitions, hints, cacheSlotWithSlotName, - tableSample); + tableSample, directMvScan); } public LogicalOlapScan withSelectedTabletIds(List selectedTabletIds) { @@ -253,7 +264,7 @@ public class LogicalOlapScan extends LogicalCatalogRelation implements OlapScan Optional.empty(), Optional.of(getLogicalProperties()), selectedPartitionIds, partitionPruned, selectedTabletIds, selectedIndexId, indexSelected, preAggStatus, manuallySpecifiedPartitions, - hints, cacheSlotWithSlotName, tableSample); + hints, cacheSlotWithSlotName, tableSample, directMvScan); } public LogicalOlapScan withPreAggStatus(PreAggStatus preAggStatus) { @@ -261,7 +272,7 @@ public class LogicalOlapScan extends LogicalCatalogRelation implements OlapScan Optional.empty(), Optional.of(getLogicalProperties()), selectedPartitionIds, partitionPruned, selectedTabletIds, selectedIndexId, indexSelected, preAggStatus, manuallySpecifiedPartitions, - hints, cacheSlotWithSlotName, tableSample); + hints, cacheSlotWithSlotName, tableSample, directMvScan); } @Override @@ -359,4 +370,8 @@ public class LogicalOlapScan extends LogicalCatalogRelation implements OlapScan public Optional getTableSample() { return tableSample; } + + public boolean isDirectMvScan() { + return directMvScan; + } } diff --git a/regression-test/data/statistics/test_select_mv.out b/regression-test/data/statistics/test_select_mv.out new file mode 100644 index 0000000000..75b7b8ddc8 --- /dev/null +++ b/regression-test/data/statistics/test_select_mv.out @@ -0,0 +1,47 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !dup_sql1 -- +4 + +-- !dup_sql2 -- +1 +2 + +-- !dup_sql3 -- +2 + +-- !dup_sql4 -- +1 2 2 3 + +-- !dup_sql5 -- +2 +4 + +-- !dup_sql6 -- +2 + +-- !dup_sql7 -- +2 4 2 6 + +-- !agg_sql1 -- +2 + +-- !agg_sql2 -- +1 +2 + +-- !agg_sql3 -- +2 + +-- !agg_sql4 -- +1 2 2 3 + +-- !agg_sql5 -- +2 +4 + +-- !agg_sql6 -- +2 + +-- !agg_sql7 -- +2 4 2 6 + diff --git a/regression-test/suites/statistics/test_select_mv.groovy b/regression-test/suites/statistics/test_select_mv.groovy new file mode 100644 index 0000000000..a35adaeb9f --- /dev/null +++ b/regression-test/suites/statistics/test_select_mv.groovy @@ -0,0 +1,97 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_select_mv") { + + def dup_sql1 = """select count(*) from test_dup;""" + def dup_sql2 = """select mv_key2 from test_dup index dup1 order by mv_key2;""" + def dup_sql3 = """select count(mv_key2) from test_dup index dup1;""" + def dup_sql4 = """select min(mv_key2), max(mv_key2), count(mv_key2), sum(mv_key2) from test_dup index dup1;""" + def dup_sql5 = """select `mva_SUM__CAST(value AS BIGINT)` as a from test_dup index dup1 order by a;""" + def dup_sql6 = """select count(`mva_SUM__CAST(value AS BIGINT)`) from test_dup index dup1;""" + def dup_sql7 = """select min(`mva_SUM__CAST(value AS BIGINT)`), max(`mva_SUM__CAST(value AS BIGINT)`), ndv(`mva_SUM__CAST(value AS BIGINT)`), sum(`mva_SUM__CAST(value AS BIGINT)`) from test_dup index dup1;""" + + def agg_sql1 = """select count(*) from test_agg;""" + def agg_sql2 = """select mv_key2 from test_agg index agg1 order by mv_key2;""" + def agg_sql3 = """select count(mv_key2) from test_agg index agg1;""" + def agg_sql4 = """select min(mv_key2), max(mv_key2), count(mv_key2), sum(mv_key2) from test_agg index agg1;""" + def agg_sql5 = """select `mva_SUM__CAST(value AS BIGINT)` as a from test_agg index agg1 order by a;""" + def agg_sql6 = """select count(`mva_SUM__CAST(value AS BIGINT)`) from test_agg index agg1;""" + def agg_sql7 = """select min(`mva_SUM__CAST(value AS BIGINT)`), max(`mva_SUM__CAST(value AS BIGINT)`), ndv(`mva_SUM__CAST(value AS BIGINT)`), sum(`mva_SUM__CAST(value AS BIGINT)`) from test_agg index agg1;""" + + + sql """drop database if exists test_select_mv""" + sql """create database test_select_mv""" + sql """use test_select_mv""" + + sql """CREATE TABLE test_dup ( + key1 int NOT NULL, + key2 int NOT NULL, + value int NOT NULL + )ENGINE=OLAP + DUPLICATE KEY(`key1`, `key2`) + DISTRIBUTED BY HASH(`key1`) BUCKETS 1 + PROPERTIES ( + "replication_num" = "1" + ); + """ + + sql """ + create materialized view dup1 as select key2, sum(value) from test_dup group by key2; + """ + + sql """CREATE TABLE test_agg ( + key1 int NOT NULL, + key2 int NOT NULL, + value int SUM NOT NULL + )ENGINE=OLAP + AGGREGATE KEY(`key1`, `key2`) + DISTRIBUTED BY HASH(`key1`) BUCKETS 1 + PROPERTIES ( + "replication_num" = "1" + ); + """ + + sql """ + create materialized view agg1 as select key2, sum(value) from test_agg group by key2; + """ + Thread.sleep(1000) + + sql """insert into test_dup values (1, 1, 1), (2, 2, 2)""" + sql """insert into test_dup values (1, 1, 1), (2, 2, 2)""" + sql """insert into test_agg values (1, 1, 1), (2, 2, 2)""" + sql """insert into test_agg values (1, 1, 1), (2, 2, 2)""" + + qt_dup_sql1 dup_sql1 + qt_dup_sql2 dup_sql2 + qt_dup_sql3 dup_sql3 + qt_dup_sql4 dup_sql4 + qt_dup_sql5 dup_sql5 + qt_dup_sql6 dup_sql6 + qt_dup_sql7 dup_sql7 + + qt_agg_sql1 agg_sql1 + qt_agg_sql2 agg_sql2 + qt_agg_sql3 agg_sql3 + qt_agg_sql4 agg_sql4 + qt_agg_sql5 agg_sql5 + qt_agg_sql6 agg_sql6 + qt_agg_sql7 agg_sql7 + + sql """drop database if exists test_select_mv""" +} +