From 8599e8ee6491221713c4445e72f9a27dbf8cd8bf Mon Sep 17 00:00:00 2001 From: seawinde <149132972+seawinde@users.noreply.github.com> Date: Tue, 28 May 2024 11:11:56 +0800 Subject: [PATCH] [improvement](mtmv) Add id to statistics map in statement context for cost estimation later (#35436) Add id to statistics map in statement context for cost estimation later this helps to improve the probability to use materialized view when query a single table with aggregate and many filter --- .../java/org/apache/doris/mtmv/MTMVCache.java | 14 ++- .../doris/nereids/StatementContext.java | 25 ++++ ...AbstractMaterializedViewAggregateRule.java | 6 +- .../mv/AbstractMaterializedViewJoinRule.java | 4 +- .../mv/AbstractMaterializedViewRule.java | 17 ++- .../mv/AsyncMaterializationContext.java | 39 ++++-- .../mv/MaterializationContext.java | 117 ++++++++++-------- .../mv/MaterializedViewScanRule.java | 4 +- .../doris/nereids/mv/IdStatisticsMapTest.java | 85 +++++++++++++ .../{memo => mv}/MvTableIdIsLongTest.java | 2 +- 10 files changed, 241 insertions(+), 72 deletions(-) create mode 100644 fe/fe-core/src/test/java/org/apache/doris/nereids/mv/IdStatisticsMapTest.java rename fe/fe-core/src/test/java/org/apache/doris/nereids/{memo => mv}/MvTableIdIsLongTest.java (98%) diff --git a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVCache.java b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVCache.java index 8bd87e2e14..c65125de2f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVCache.java +++ b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVCache.java @@ -34,6 +34,7 @@ import org.apache.doris.nereids.trees.plans.logical.LogicalResultSink; import org.apache.doris.nereids.trees.plans.visitor.DefaultPlanRewriter; import org.apache.doris.qe.ConnectContext; import org.apache.doris.qe.OriginStatement; +import org.apache.doris.statistics.Statistics; import com.google.common.collect.ImmutableList; @@ -47,10 +48,12 @@ public class MTMVCache { private final Plan logicalPlan; // The original plan of mv def sql private final Plan originalPlan; + private final Statistics statistics; - public MTMVCache(Plan logicalPlan, Plan originalPlan) { + public MTMVCache(Plan logicalPlan, Plan originalPlan, Statistics statistics) { this.logicalPlan = logicalPlan; this.originalPlan = originalPlan; + this.statistics = statistics; } public Plan getLogicalPlan() { @@ -61,6 +64,10 @@ public class MTMVCache { return originalPlan; } + public Statistics getStatistics() { + return statistics; + } + public static MTMVCache from(MTMV mtmv, ConnectContext connectContext) { LogicalPlan unboundMvPlan = new NereidsParser().parseSingle(mtmv.getQuerySql()); StatementContext mvSqlStatementContext = new StatementContext(connectContext, @@ -71,7 +78,8 @@ public class MTMVCache { } // Can not convert to table sink, because use the same column from different table when self join // the out slot is wrong - Plan originPlan = planner.plan(unboundMvPlan, PhysicalProperties.ANY, ExplainLevel.REWRITTEN_PLAN); + planner.plan(unboundMvPlan, PhysicalProperties.ANY, ExplainLevel.ALL_PLAN); + Plan originPlan = planner.getCascadesContext().getRewritePlan(); // Eliminate result sink because sink operator is useless in query rewrite by materialized view // and the top sort can also be removed Plan mvPlan = originPlan.accept(new DefaultPlanRewriter() { @@ -88,6 +96,6 @@ public class MTMVCache { ImmutableList.of(Rewriter.custom(RuleType.ELIMINATE_SORT, EliminateSort::new))).execute(); return childContext.getRewritePlan(); }, mvPlan, originPlan); - return new MTMVCache(mvPlan, originPlan); + return new MTMVCache(mvPlan, originPlan, planner.getCascadesContext().getMemo().getRoot().getStatistics()); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/StatementContext.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/StatementContext.java index 6e4b3c23fc..f4a32d723a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/StatementContext.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/StatementContext.java @@ -20,6 +20,7 @@ package org.apache.doris.nereids; import org.apache.doris.analysis.StatementBase; import org.apache.doris.catalog.TableIf; import org.apache.doris.catalog.constraint.TableIdentifier; +import org.apache.doris.common.Id; import org.apache.doris.common.IdGenerator; import org.apache.doris.common.Pair; import org.apache.doris.nereids.hint.Hint; @@ -41,7 +42,9 @@ import org.apache.doris.qe.ConnectContext; import org.apache.doris.qe.OriginStatement; import org.apache.doris.qe.SessionVariable; import org.apache.doris.qe.cache.CacheAnalyzer; +import org.apache.doris.statistics.Statistics; +import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Stopwatch; import com.google.common.base.Supplier; import com.google.common.base.Suppliers; @@ -146,6 +149,10 @@ public class StatementContext implements Closeable { // Record table id mapping, the key is the hash code of union catalogId, databaseId, tableId // the value is the auto-increment id in the cascades context private final Map tableIdMapping = new LinkedHashMap<>(); + // Record the materialization statistics by id which is used for cost estimation. + // Maybe return null, which means the id according statistics should calc normally rather than getting + // form this map + private final Map relationIdToStatisticsMap = new LinkedHashMap<>(); public StatementContext() { this(ConnectContext.get(), null, 0); @@ -412,6 +419,24 @@ public class StatementContext implements Closeable { indexInSqlToString.put(pair, replacement); } + public void addStatistics(Id id, Statistics statistics) { + if (id instanceof RelationId) { + this.relationIdToStatisticsMap.put((RelationId) id, statistics); + } + } + + public Optional getStatistics(Id id) { + if (id instanceof RelationId) { + return Optional.ofNullable(this.relationIdToStatisticsMap.get((RelationId) id)); + } + return Optional.empty(); + } + + @VisibleForTesting + public Map getRelationIdToStatisticsMap() { + return relationIdToStatisticsMap; + } + /** addTableReadLock */ public synchronized void addTableReadLock(TableIf tableIf) { if (!tableIf.needReadLockWhenPlan()) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewAggregateRule.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewAggregateRule.java index b37b04d802..1e013498a1 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewAggregateRule.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewAggregateRule.java @@ -97,7 +97,7 @@ public abstract class AbstractMaterializedViewAggregateRule extends AbstractMate viewStructInfo)) { List rewrittenQueryExpressions = rewriteExpression(queryTopPlan.getOutput(), queryTopPlan, - materializationContext.getMvExprToMvScanExprMapping(), + materializationContext.getExprToScanExprMapping(), viewToQuerySlotMapping, true, queryStructInfo.getTableBitSet()); @@ -121,7 +121,7 @@ public abstract class AbstractMaterializedViewAggregateRule extends AbstractMate () -> String.format("expressionToWrite = %s,\n mvExprToMvScanExprMapping = %s,\n" + "viewToQuerySlotMapping = %s", queryTopPlan.getOutput(), - materializationContext.getMvExprToMvScanExprMapping(), + materializationContext.getExprToScanExprMapping(), viewToQuerySlotMapping)); } // if view is scalar aggregate but query is not. Or if query is scalar aggregate but view is not @@ -150,7 +150,7 @@ public abstract class AbstractMaterializedViewAggregateRule extends AbstractMate List queryExpressions = queryTopPlan.getOutput(); // permute the mv expr mapping to query based Map mvExprToMvScanExprQueryBased = - materializationContext.getMvExprToMvScanExprMapping().keyPermute(viewToQuerySlotMapping) + materializationContext.getExprToScanExprMapping().keyPermute(viewToQuerySlotMapping) .flattenMap().get(0); for (Expression topExpression : queryExpressions) { // if agg function, try to roll up and rewrite diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewJoinRule.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewJoinRule.java index 3b20cefbba..4f95c248ec 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewJoinRule.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewJoinRule.java @@ -46,7 +46,7 @@ public abstract class AbstractMaterializedViewJoinRule extends AbstractMateriali List expressionsRewritten = rewriteExpression( queryStructInfo.getExpressions(), queryStructInfo.getTopPlan(), - materializationContext.getMvExprToMvScanExprMapping(), + materializationContext.getExprToScanExprMapping(), targetToSourceMapping, true, queryStructInfo.getTableBitSet() @@ -57,7 +57,7 @@ public abstract class AbstractMaterializedViewJoinRule extends AbstractMateriali "Rewrite expressions by view in join fail", () -> String.format("expressionToRewritten is %s,\n mvExprToMvScanExprMapping is %s,\n" + "targetToSourceMapping = %s", queryStructInfo.getExpressions(), - materializationContext.getMvExprToMvScanExprMapping(), + materializationContext.getExprToScanExprMapping(), targetToSourceMapping)); return null; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewRule.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewRule.java index 8442d2485c..ea057e8bfc 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewRule.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewRule.java @@ -24,6 +24,7 @@ import org.apache.doris.catalog.PartitionInfo; import org.apache.doris.catalog.PartitionItem; import org.apache.doris.catalog.PartitionType; import org.apache.doris.catalog.TableIf; +import org.apache.doris.common.Id; import org.apache.doris.common.Pair; import org.apache.doris.mtmv.BaseTableInfo; import org.apache.doris.mtmv.MTMVPartitionInfo; @@ -59,6 +60,7 @@ import org.apache.doris.nereids.trees.plans.logical.LogicalProject; import org.apache.doris.nereids.trees.plans.logical.LogicalUnion; import org.apache.doris.nereids.util.ExpressionUtils; import org.apache.doris.nereids.util.TypeUtils; +import org.apache.doris.statistics.Statistics; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMultimap; @@ -77,6 +79,7 @@ import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.Objects; +import java.util.Optional; import java.util.Set; import java.util.stream.Collectors; @@ -226,21 +229,21 @@ public abstract class AbstractMaterializedViewRule implements ExplorationRuleFac continue; } Plan rewrittenPlan; - Plan mvScan = materializationContext.getMvScanPlan(); + Plan mvScan = materializationContext.getScanPlan(); Plan queryPlan = queryStructInfo.getTopPlan(); if (compensatePredicates.isAlwaysTrue()) { rewrittenPlan = mvScan; } else { // Try to rewrite compensate predicates by using mv scan List rewriteCompensatePredicates = rewriteExpression(compensatePredicates.toList(), - queryPlan, materializationContext.getMvExprToMvScanExprMapping(), + queryPlan, materializationContext.getExprToScanExprMapping(), viewToQuerySlotMapping, true, queryStructInfo.getTableBitSet()); if (rewriteCompensatePredicates.isEmpty()) { materializationContext.recordFailReason(queryStructInfo, "Rewrite compensate predicate by view fail", () -> String.format("compensatePredicates = %s,\n mvExprToMvScanExprMapping = %s,\n" + "viewToQuerySlotMapping = %s", - compensatePredicates, materializationContext.getMvExprToMvScanExprMapping(), + compensatePredicates, materializationContext.getExprToScanExprMapping(), viewToQuerySlotMapping)); continue; } @@ -334,9 +337,15 @@ public abstract class AbstractMaterializedViewRule implements ExplorationRuleFac continue; } recordIfRewritten(queryStructInfo.getOriginalPlan(), materializationContext); + Optional> materializationPlanStatistics = + materializationContext.getPlanStatistics(cascadesContext); + if (materializationPlanStatistics.isPresent() && materializationPlanStatistics.get().key() != null) { + cascadesContext.getStatementContext().addStatistics( + materializationPlanStatistics.get().key(), materializationPlanStatistics.get().value()); + } rewriteResults.add(rewrittenPlan); // if rewrite successfully, try to regenerate mv scan because it maybe used again - materializationContext.tryReGenerateMvScanPlan(cascadesContext); + materializationContext.tryReGenerateScanPlan(cascadesContext); } return rewriteResults; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AsyncMaterializationContext.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AsyncMaterializationContext.java index 1c0d854dd9..f1c4372f10 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AsyncMaterializationContext.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AsyncMaterializationContext.java @@ -19,14 +19,20 @@ package org.apache.doris.nereids.rules.exploration.mv; import org.apache.doris.catalog.MTMV; import org.apache.doris.catalog.Table; +import org.apache.doris.common.AnalysisException; +import org.apache.doris.common.Id; import org.apache.doris.common.Pair; +import org.apache.doris.mtmv.MTMVCache; import org.apache.doris.nereids.CascadesContext; import org.apache.doris.nereids.rules.exploration.mv.mapping.ExpressionMapping; import org.apache.doris.nereids.trees.plans.ObjectId; import org.apache.doris.nereids.trees.plans.Plan; +import org.apache.doris.nereids.trees.plans.RelationId; import org.apache.doris.nereids.trees.plans.algebra.Relation; +import org.apache.doris.nereids.trees.plans.logical.LogicalOlapScan; import org.apache.doris.nereids.trees.plans.physical.PhysicalCatalogRelation; import org.apache.doris.nereids.util.Utils; +import org.apache.doris.statistics.Statistics; import com.google.common.collect.Multimap; import org.apache.logging.log4j.LogManager; @@ -35,6 +41,7 @@ import org.apache.logging.log4j.Logger; import java.util.Collection; import java.util.List; import java.util.Map; +import java.util.Optional; /** * Async context for query rewrite by materialized view @@ -58,7 +65,7 @@ public class AsyncMaterializationContext extends MaterializationContext { } @Override - Plan doGenerateMvPlan(CascadesContext cascadesContext) { + Plan doGenerateScanPlan(CascadesContext cascadesContext) { return MaterializedViewUtils.generateMvScanPlan(this.mtmv, cascadesContext); } @@ -85,6 +92,24 @@ public class AsyncMaterializationContext extends MaterializationContext { "failReason", failReasonBuilder.toString()); } + @Override + public Optional> getPlanStatistics(CascadesContext cascadesContext) { + MTMVCache mtmvCache; + try { + mtmvCache = mtmv.getOrGenerateCache(cascadesContext.getConnectContext()); + } catch (AnalysisException e) { + LOG.warn(String.format("get mv plan statistics fail, materialization qualifier is %s", + getMaterializationQualifier()), e); + return Optional.empty(); + } + RelationId relationId = null; + List scanObjs = this.getPlan().collectFirst(plan -> plan instanceof LogicalOlapScan); + if (scanObjs != null && !scanObjs.isEmpty()) { + relationId = ((LogicalOlapScan) scanObjs.get(0)).getRelationId(); + } + return Optional.of(Pair.of(relationId, mtmvCache.getStatistics())); + } + @Override boolean isFinalChosen(Relation relation) { if (!(relation instanceof PhysicalCatalogRelation)) { @@ -93,8 +118,8 @@ public class AsyncMaterializationContext extends MaterializationContext { return ((PhysicalCatalogRelation) relation).getTable() instanceof MTMV; } - public Plan getMvScanPlan() { - return mvScanPlan; + public Plan getScanPlan() { + return scanPlan; } public List getBaseTables() { @@ -105,16 +130,16 @@ public class AsyncMaterializationContext extends MaterializationContext { return baseViews; } - public ExpressionMapping getMvExprToMvScanExprMapping() { - return mvExprToMvScanExprMapping; + public ExpressionMapping getExprToScanExprMapping() { + return exprToScanExprMapping; } public boolean isAvailable() { return available; } - public Plan getMvPlan() { - return mvPlan; + public Plan getPlan() { + return plan; } public Multimap> getFailReason() { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializationContext.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializationContext.java index 13e8c4456c..d67390476e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializationContext.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializationContext.java @@ -19,6 +19,7 @@ package org.apache.doris.nereids.rules.exploration.mv; import org.apache.doris.analysis.StatementBase; import org.apache.doris.catalog.Table; +import org.apache.doris.common.Id; import org.apache.doris.common.Pair; import org.apache.doris.nereids.CascadesContext; import org.apache.doris.nereids.memo.GroupId; @@ -34,6 +35,7 @@ import org.apache.doris.nereids.trees.plans.physical.PhysicalPlan; import org.apache.doris.nereids.trees.plans.physical.PhysicalRelation; import org.apache.doris.nereids.trees.plans.visitor.DefaultPlanVisitor; import org.apache.doris.nereids.util.ExpressionUtils; +import org.apache.doris.statistics.Statistics; import com.google.common.collect.HashMultimap; import com.google.common.collect.ImmutableSet; @@ -47,6 +49,7 @@ import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Objects; +import java.util.Optional; import java.util.Set; import java.util.function.Supplier; import java.util.stream.Collectors; @@ -56,69 +59,75 @@ import java.util.stream.Collectors; */ public abstract class MaterializationContext { private static final Logger LOG = LogManager.getLogger(MaterializationContext.class); - public final Map queryToMvSlotMappingCache = new HashMap<>(); + public final Map queryToMaterializationSlotMappingCache = new HashMap<>(); protected List
baseTables; protected List
baseViews; - // The plan of mv def sql - protected Plan mvPlan; - // The original plan of mv def sql - protected Plan originalMvPlan; + // The plan of materialization def sql + protected Plan plan; + // The original plan of materialization sql + protected Plan originalPlan; // Should regenerate when materialization is already rewritten successfully because one query may hit repeatedly // make sure output is different in multi using - protected Plan mvScanPlan; - // The mvPlan output shuttled expression, this is used by generate field mvExprToMvScanExprMapping - protected List mvPlanOutputShuttledExpressions; - // Generated mapping from mv plan out shuttled expr to mv scan plan out slot mapping, this is used for later used - protected ExpressionMapping mvExprToMvScanExprMapping; + protected Plan scanPlan; + // The materialization plan output shuttled expression, this is used by generate field + // exprToScanExprMapping + protected List planOutputShuttledExpressions; + // Generated mapping from materialization plan out shuttled expr to materialization scan plan out slot mapping, + // this is used for later used + protected ExpressionMapping exprToScanExprMapping; // This mark the materialization context is available or not, // will not be used in query transparent rewritten if false protected boolean available = true; - // Mark the mv plan in the context is already rewritten successfully or not + // Mark the materialization plan in the context is already rewritten successfully or not protected boolean success = false; // Mark enable record failure detail info or not, because record failure detail info is performance-depleting protected final boolean enableRecordFailureDetail; - // The mv plan struct info + // The materialization plan struct info protected final StructInfo structInfo; - // Group id set that are rewritten unsuccessfully by this mv for reducing rewrite times + // Group id set that are rewritten unsuccessfully by this materialization for reducing rewrite times protected final Set matchedFailGroups = new HashSet<>(); - // Group id set that are rewritten successfully by this mv for reducing rewrite times + // Group id set that are rewritten successfully by this materialization for reducing rewrite times protected final Set matchedSuccessGroups = new HashSet<>(); - // Record the reason, if rewrite by mv fail. The failReason should be empty if success. + // Record the reason, if rewrite by materialization fail. The failReason should be empty if success. // The key is the query belonged group expression objectId, the value is the fail reasons because // for one materialization query may be multi when nested materialized view. protected final Multimap> failReason = HashMultimap.create(); /** - * MaterializationContext, this contains necessary info for query rewriting by mv + * MaterializationContext, this contains necessary info for query rewriting by materialization */ - public MaterializationContext(Plan mvPlan, Plan originalMvPlan, Plan mvScanPlan, CascadesContext cascadesContext) { - this.mvPlan = mvPlan; - this.originalMvPlan = originalMvPlan; - this.mvScanPlan = mvScanPlan; + public MaterializationContext(Plan plan, Plan originalPlan, + Plan scanPlan, CascadesContext cascadesContext) { + this.plan = plan; + this.originalPlan = originalPlan; + this.scanPlan = scanPlan; StatementBase parsedStatement = cascadesContext.getStatementContext().getParsedStatement(); this.enableRecordFailureDetail = parsedStatement != null && parsedStatement.isExplain() && ExplainLevel.MEMO_PLAN == parsedStatement.getExplainOptions().getExplainLevel(); - this.mvPlanOutputShuttledExpressions = ExpressionUtils.shuttleExpressionWithLineage( - originalMvPlan.getOutput(), - originalMvPlan, + this.planOutputShuttledExpressions = ExpressionUtils.shuttleExpressionWithLineage( + originalPlan.getOutput(), + originalPlan, new BitSet()); - // mv output expression shuttle, this will be used to expression rewrite - this.mvExprToMvScanExprMapping = ExpressionMapping.generate(this.mvPlanOutputShuttledExpressions, - this.mvScanPlan.getOutput()); - // Construct mv struct info, catch exception which may cause planner roll back + // materialization output expression shuttle, this will be used to expression rewrite + this.exprToScanExprMapping = ExpressionMapping.generate( + this.planOutputShuttledExpressions, + this.scanPlan.getOutput()); + // Construct materialization struct info, catch exception which may cause planner roll back List viewStructInfos; try { - viewStructInfos = MaterializedViewUtils.extractStructInfo(mvPlan, cascadesContext, new BitSet()); + viewStructInfos = MaterializedViewUtils.extractStructInfo(plan, cascadesContext, new BitSet()); if (viewStructInfos.size() > 1) { // view struct info should only have one, log error and use the first struct info - LOG.warn(String.format("view strut info is more than one, mv scan plan is %s, mv plan is %s", - mvScanPlan.treeString(), mvPlan.treeString())); + LOG.warn(String.format("view strut info is more than one, materialization scan plan is %s, " + + "materialization plan is %s", + scanPlan.treeString(), plan.treeString())); } } catch (Exception exception) { - LOG.warn(String.format("construct mv struct info fail, mv scan plan is %s, mv plan is %s", - mvScanPlan.treeString(), mvPlan.treeString()), exception); + LOG.warn(String.format("construct materialization struct info fail, materialization scan plan is %s, " + + "materialization plan is %s", + scanPlan.treeString(), plan.treeString()), exception); this.available = false; this.structInfo = null; return; @@ -141,32 +150,33 @@ public abstract class MaterializationContext { /** * Try to generate scan plan for materialization * if MaterializationContext is already rewritten successfully, then should generate new scan plan in later - * query rewrite, because one plan may hit the materialized view repeatedly and the mv scan output + * query rewrite, because one plan may hit the materialized view repeatedly and the materialization scan output * should be different. * This method should be called when query rewrite successfully */ - public void tryReGenerateMvScanPlan(CascadesContext cascadesContext) { - this.mvScanPlan = doGenerateMvPlan(cascadesContext); - // mv output expression shuttle, this will be used to expression rewrite - this.mvExprToMvScanExprMapping = ExpressionMapping.generate(this.mvPlanOutputShuttledExpressions, - this.mvScanPlan.getOutput()); + public void tryReGenerateScanPlan(CascadesContext cascadesContext) { + this.scanPlan = doGenerateScanPlan(cascadesContext); + // materialization output expression shuttle, this will be used to expression rewrite + this.exprToScanExprMapping = ExpressionMapping.generate( + this.planOutputShuttledExpressions, + this.scanPlan.getOutput()); } public void addSlotMappingToCache(RelationMapping relationMapping, SlotMapping slotMapping) { - queryToMvSlotMappingCache.put(relationMapping, slotMapping); + queryToMaterializationSlotMappingCache.put(relationMapping, slotMapping); } public SlotMapping getSlotMappingFromCache(RelationMapping relationMapping) { - return queryToMvSlotMappingCache.get(relationMapping); + return queryToMaterializationSlotMappingCache.get(relationMapping); } /** * Try to generate scan plan for materialization * if MaterializationContext is already rewritten successfully, then should generate new scan plan in later - * query rewrite, because one plan may hit the materialized view repeatedly and the mv scan output + * query rewrite, because one plan may hit the materialized view repeatedly and the materialization scan output * should be different */ - abstract Plan doGenerateMvPlan(CascadesContext cascadesContext); + abstract Plan doGenerateScanPlan(CascadesContext cascadesContext); /** * Get materialization unique qualifier which identify it @@ -178,21 +188,28 @@ public abstract class MaterializationContext { */ abstract String getStringInfo(); + /** + * Get materialization plan statistics, the key is the identifier of statistics + * the value is Statistics. + * the statistics is used by cost estimation when the materialization is used + */ + abstract Optional> getPlanStatistics(CascadesContext cascadesContext); + /** * Calc the relation is chosen finally or not */ abstract boolean isFinalChosen(Relation relation); - public Plan getMvPlan() { - return mvPlan; + public Plan getPlan() { + return plan; } - public Plan getOriginalMvPlan() { - return originalMvPlan; + public Plan getOriginalPlan() { + return originalPlan; } - public Plan getMvScanPlan() { - return mvScanPlan; + public Plan getScanPlan() { + return scanPlan; } public List
getBaseTables() { @@ -203,8 +220,8 @@ public abstract class MaterializationContext { return baseViews; } - public ExpressionMapping getMvExprToMvScanExprMapping() { - return mvExprToMvScanExprMapping; + public ExpressionMapping getExprToScanExprMapping() { + return exprToScanExprMapping; } public boolean isAvailable() { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewScanRule.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewScanRule.java index d6d7817d35..82e7944a81 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewScanRule.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewScanRule.java @@ -47,7 +47,7 @@ public abstract class MaterializedViewScanRule extends AbstractMaterializedViewR List expressionsRewritten = rewriteExpression( queryStructInfo.getExpressions(), queryStructInfo.getTopPlan(), - materializationContext.getMvExprToMvScanExprMapping(), + materializationContext.getExprToScanExprMapping(), targetToSourceMapping, true, queryStructInfo.getTableBitSet() @@ -58,7 +58,7 @@ public abstract class MaterializedViewScanRule extends AbstractMaterializedViewR "Rewrite expressions by view in scan fail", () -> String.format("expressionToRewritten is %s,\n mvExprToMvScanExprMapping is %s,\n" + "targetToSourceMapping = %s", queryStructInfo.getExpressions(), - materializationContext.getMvExprToMvScanExprMapping(), + materializationContext.getExprToScanExprMapping(), targetToSourceMapping)); return null; } diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/mv/IdStatisticsMapTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/mv/IdStatisticsMapTest.java new file mode 100644 index 0000000000..6660457b88 --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/mv/IdStatisticsMapTest.java @@ -0,0 +1,85 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.mv; + +import org.apache.doris.catalog.MTMV; +import org.apache.doris.mtmv.MTMVRelationManager; +import org.apache.doris.nereids.CascadesContext; +import org.apache.doris.nereids.sqltest.SqlTestBase; +import org.apache.doris.nereids.trees.plans.RelationId; +import org.apache.doris.nereids.util.PlanChecker; +import org.apache.doris.qe.ConnectContext; +import org.apache.doris.qe.SessionVariable; +import org.apache.doris.statistics.Statistics; + +import mockit.Mock; +import mockit.MockUp; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.util.BitSet; +import java.util.Map; +import java.util.Optional; + +/** + * Test idStatisticsMap in StatementContext is valid + */ +public class IdStatisticsMapTest extends SqlTestBase { + + @Test + void testIdStatisticsIsExistWhenRewriteByMv() throws Exception { + connectContext.getSessionVariable().setDisableNereidsRules("PRUNE_EMPTY_PARTITION"); + BitSet disableNereidsRules = connectContext.getSessionVariable().getDisableNereidsRules(); + new MockUp() { + @Mock + public BitSet getDisableNereidsRules() { + return disableNereidsRules; + } + }; + new MockUp() { + @Mock + public boolean isMVPartitionValid(MTMV mtmv, ConnectContext ctx) { + return true; + } + }; + connectContext.getSessionVariable().enableMaterializedViewRewrite = true; + connectContext.getSessionVariable().enableMaterializedViewNestRewrite = true; + createMvByNereids("create materialized view mv100 BUILD IMMEDIATE REFRESH COMPLETE ON MANUAL\n" + + " DISTRIBUTED BY RANDOM BUCKETS 1\n" + + " PROPERTIES ('replication_num' = '1') \n" + + " as select T1.id from T1 inner join T2 " + + "on T1.id = T2.id;"); + CascadesContext c1 = createCascadesContext( + "select T1.id from T1 inner join T2 " + + "on T1.id = T2.id " + + "inner join T3 on T1.id = T3.id", + connectContext + ); + PlanChecker.from(c1) + .analyze() + .rewrite() + .optimize() + .printlnBestPlanTree(); + Map idStatisticsMap = c1.getStatementContext().getRelationIdToStatisticsMap(); + Assertions.assertFalse(idStatisticsMap.isEmpty()); + RelationId relationId = idStatisticsMap.keySet().iterator().next(); + Optional statistics = c1.getStatementContext().getStatistics(relationId); + Assertions.assertTrue(statistics.isPresent()); + dropMvByNereids("drop materialized view mv100"); + } +} diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/memo/MvTableIdIsLongTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/mv/MvTableIdIsLongTest.java similarity index 98% rename from fe/fe-core/src/test/java/org/apache/doris/nereids/memo/MvTableIdIsLongTest.java rename to fe/fe-core/src/test/java/org/apache/doris/nereids/mv/MvTableIdIsLongTest.java index 75aa7718c0..fd3887d3cf 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/memo/MvTableIdIsLongTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/mv/MvTableIdIsLongTest.java @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -package org.apache.doris.nereids.memo; +package org.apache.doris.nereids.mv; import org.apache.doris.catalog.MTMV; import org.apache.doris.mtmv.MTMVRelationManager;