database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[refactor](mtmv) Materialization context and mtmv decoupling (#34093) (#34916)

Browse Source 
Decoupling the MTMV from the materialization context.
Change MaterializationContext to abstract which is the materialization desc.
It now has AsyncMaterializationContext sub class, can also has other type of MaterializationContext such as
SyncMaterializationContext and so on.
This commit is contained in:
seawinde
2024-05-17 22:54:21 +08:00
committed by GitHub
parent 385739564d
commit b76cfcd007
8 changed files with 351 additions and 171 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVCache.java
View File

@ -42,9 +42,10 @@ import com.google.common.collect.ImmutableList;
*/
public class MTMVCache {
// the materialized view plan which should be optimized by the same rules to query
// The materialized view plan which should be optimized by the same rules to query
// and will remove top sink and unused sort
private final Plan logicalPlan;
// for stable output order, we should use original plan
// The original plan of mv def sql
private final Plan originalPlan;
public MTMVCache(Plan logicalPlan, Plan originalPlan) {

16
fe/fe-core/src/main/java/org/apache/doris/nereids/NereidsPlanner.java
View File

@ -23,7 +23,6 @@ import org.apache.doris.analysis.LiteralExpr;
import org.apache.doris.analysis.StatementBase;
import org.apache.doris.catalog.Column;
import org.apache.doris.catalog.Env;
import org.apache.doris.catalog.MTMV;
import org.apache.doris.common.NereidsException;
import org.apache.doris.common.Pair;
import org.apache.doris.common.profile.SummaryProfile;
@ -53,7 +52,6 @@ import org.apache.doris.nereids.trees.plans.Plan;
import org.apache.doris.nereids.trees.plans.commands.ExplainCommand.ExplainLevel;
import org.apache.doris.nereids.trees.plans.logical.LogicalPlan;
import org.apache.doris.nereids.trees.plans.logical.LogicalSqlCache;
import org.apache.doris.nereids.trees.plans.physical.PhysicalCatalogRelation;
import org.apache.doris.nereids.trees.plans.physical.PhysicalEmptyRelation;
import org.apache.doris.nereids.trees.plans.physical.PhysicalOneRowRelation;
import org.apache.doris.nereids.trees.plans.physical.PhysicalPlan;
@ -467,11 +465,16 @@ public class NereidsPlanner extends Planner {
plan = optimizedPlan.shape("");
break;
case MEMO_PLAN:
StringBuilder materializationStringBuilder = new StringBuilder();
materializationStringBuilder.append("materializationContexts:").append("\n");
for (MaterializationContext ctx : cascadesContext.getMaterializationContexts()) {
materializationStringBuilder.append("\n").append(ctx).append("\n");
}
plan = cascadesContext.getMemo().toString()
+ "\n\n========== OPTIMIZED PLAN ==========\n"
+ optimizedPlan.treeString()
+ "\n\n========== MATERIALIZATIONS ==========\n"
+ MaterializationContext.toDetailString(cascadesContext.getMaterializationContexts());
+ materializationStringBuilder;
break;
case ALL_PLAN:
plan = "========== PARSED PLAN "
@ -488,14 +491,9 @@ public class NereidsPlanner extends Planner {
+ optimizedPlan.treeString();
break;
default:
List<MTMV> materializationListChosenByCbo = this.getPhysicalPlan()
.collectToList(node -> node instanceof PhysicalCatalogRelation
&& ((PhysicalCatalogRelation) node).getTable() instanceof MTMV).stream()
.map(node -> (MTMV) ((PhysicalCatalogRelation) node).getTable())
.collect(Collectors.toList());
plan = super.getExplainString(explainOptions)
+ MaterializationContext.toSummaryString(cascadesContext.getMaterializationContexts(),
materializationListChosenByCbo);
this.getPhysicalPlan());
}
if (statementContext != null && !statementContext.getHints().isEmpty()) {
String hint = getHintExplainString(statementContext.getHints());

82
fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewRule.java
View File

@ -237,14 +237,14 @@ public abstract class AbstractMaterializedViewRule implements ExplorationRuleFac
// Rewrite query by view
rewrittenPlan = rewriteQueryByView(matchMode, queryStructInfo, viewStructInfo, viewToQuerySlotMapping,
rewrittenPlan, materializationContext);
if (rewrittenPlan == null) {
continue;
}
rewrittenPlan = MaterializedViewUtils.rewriteByRules(cascadesContext,
childContext -> {
Rewriter.getWholeTreeRewriter(childContext).execute();
return childContext.getRewritePlan();
}, rewrittenPlan, queryPlan);
if (rewrittenPlan == null) {
continue;
}
// check the partitions used by rewritten plan is valid or not
Multimap<Pair<MTMVPartitionInfo, PartitionInfo>, Partition> invalidPartitionsQueryUsed =
calcUsedInvalidMvPartitions(rewrittenPlan, materializationContext, cascadesContext);
@ -287,9 +287,10 @@ public abstract class AbstractMaterializedViewRule implements ExplorationRuleFac
}
// For rewrittenPlan which contains materialized view should remove invalid partition ids
List<Plan> children = Lists.newArrayList(
rewrittenPlan.accept(new InvalidPartitionRemover(), Pair.of(materializationContext.getMTMV(),
invalidPartitionsQueryUsed.values().stream()
.map(Partition::getId).collect(Collectors.toSet()))),
rewrittenPlan.accept(new InvalidPartitionRemover(), Pair.of(
materializationContext.getMaterializationQualifier(),
invalidPartitionsQueryUsed.values().stream().map(Partition::getId)
.collect(Collectors.toSet()))),
StructInfo.addFilterOnTableScan(queryPlan, filterOnOriginPlan, cascadesContext));
// Union query materialized view and source table
rewrittenPlan = new LogicalUnion(Qualifier.ALL,
@ -387,40 +388,43 @@ public abstract class AbstractMaterializedViewRule implements ExplorationRuleFac
Plan rewrittenPlan,
MaterializationContext materializationContext,
CascadesContext cascadesContext) {
// check partition is valid or not
MTMV mtmv = materializationContext.getMTMV();
PartitionInfo mvPartitionInfo = mtmv.getPartitionInfo();
if (PartitionType.UNPARTITIONED.equals(mvPartitionInfo.getType())) {
// if not partition, if rewrite success, it means mv is available
return ImmutableMultimap.of();
if (materializationContext instanceof AsyncMaterializationContext) {
// check partition is valid or not
MTMV mtmv = ((AsyncMaterializationContext) materializationContext).getMtmv();
PartitionInfo mvPartitionInfo = mtmv.getPartitionInfo();
if (PartitionType.UNPARTITIONED.equals(mvPartitionInfo.getType())) {
// if not partition, if rewrite success, it means mv is available
return ImmutableMultimap.of();
}
// check mv related table partition is valid or not
MTMVPartitionInfo mvCustomPartitionInfo = mtmv.getMvPartitionInfo();
BaseTableInfo relatedPartitionTable = mvCustomPartitionInfo.getRelatedTableInfo();
if (relatedPartitionTable == null) {
return ImmutableMultimap.of();
}
// get mv valid partitions
Set<Long> mvDataValidPartitionIdSet = MTMVRewriteUtil.getMTMVCanRewritePartitions(mtmv,
cascadesContext.getConnectContext(), System.currentTimeMillis()).stream()
.map(Partition::getId)
.collect(Collectors.toSet());
// get partitions query used
Set<Long> mvPartitionSetQueryUsed = rewrittenPlan.collectToList(node -> node instanceof LogicalOlapScan
&& Objects.equals(((CatalogRelation) node).getTable().getName(), mtmv.getName()))
.stream()
.map(node -> ((LogicalOlapScan) node).getSelectedPartitionIds())
.flatMap(Collection::stream)
.collect(Collectors.toSet());
// get invalid partition ids
Set<Long> invalidMvPartitionIdSet = new HashSet<>(mvPartitionSetQueryUsed);
invalidMvPartitionIdSet.removeAll(mvDataValidPartitionIdSet);
ImmutableMultimap.Builder<Pair<MTMVPartitionInfo, PartitionInfo>, Partition> invalidPartitionMapBuilder =
ImmutableMultimap.builder();
Pair<MTMVPartitionInfo, PartitionInfo> partitionInfo = Pair.of(mvCustomPartitionInfo, mvPartitionInfo);
invalidMvPartitionIdSet.forEach(invalidPartitionId ->
invalidPartitionMapBuilder.put(partitionInfo, mtmv.getPartition(invalidPartitionId)));
return invalidPartitionMapBuilder.build();
}
// check mv related table partition is valid or not
MTMVPartitionInfo mvCustomPartitionInfo = mtmv.getMvPartitionInfo();
BaseTableInfo relatedPartitionTable = mvCustomPartitionInfo.getRelatedTableInfo();
if (relatedPartitionTable == null) {
return ImmutableMultimap.of();
}
// get mv valid partitions
Set<Long> mvDataValidPartitionIdSet = MTMVRewriteUtil.getMTMVCanRewritePartitions(mtmv,
cascadesContext.getConnectContext(), System.currentTimeMillis()).stream()
.map(Partition::getId)
.collect(Collectors.toSet());
// get partitions query used
Set<Long> mvPartitionSetQueryUsed = rewrittenPlan.collectToList(node -> node instanceof LogicalOlapScan
&& Objects.equals(((CatalogRelation) node).getTable().getName(), mtmv.getName()))
.stream()
.map(node -> ((LogicalOlapScan) node).getSelectedPartitionIds())
.flatMap(Collection::stream)
.collect(Collectors.toSet());
// get invalid partition ids
Set<Long> invalidMvPartitionIdSet = new HashSet<>(mvPartitionSetQueryUsed);
invalidMvPartitionIdSet.removeAll(mvDataValidPartitionIdSet);
ImmutableMultimap.Builder<Pair<MTMVPartitionInfo, PartitionInfo>, Partition> invalidPartitionMapBuilder =
ImmutableMultimap.builder();
Pair<MTMVPartitionInfo, PartitionInfo> partitionInfo = Pair.of(mvCustomPartitionInfo, mvPartitionInfo);
invalidMvPartitionIdSet.forEach(invalidPartitionId ->
invalidPartitionMapBuilder.put(partitionInfo, mtmv.getPartition(invalidPartitionId)));
return invalidPartitionMapBuilder.build();
return ImmutableMultimap.of();
}
/**

140
fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AsyncMaterializationContext.java Normal file
View File

@ -0,0 +1,140 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package org.apache.doris.nereids.rules.exploration.mv;
import org.apache.doris.catalog.MTMV;
import org.apache.doris.catalog.Table;
import org.apache.doris.common.Pair;
import org.apache.doris.nereids.CascadesContext;
import org.apache.doris.nereids.rules.exploration.mv.mapping.ExpressionMapping;
import org.apache.doris.nereids.trees.plans.ObjectId;
import org.apache.doris.nereids.trees.plans.Plan;
import org.apache.doris.nereids.trees.plans.algebra.Relation;
import org.apache.doris.nereids.trees.plans.physical.PhysicalCatalogRelation;
import org.apache.doris.nereids.util.Utils;
import com.google.common.collect.Multimap;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import java.util.Collection;
import java.util.List;
import java.util.Map;
/**
* Async context for query rewrite by materialized view
*/
public class AsyncMaterializationContext extends MaterializationContext {
private static final Logger LOG = LogManager.getLogger(AsyncMaterializationContext.class);
private final MTMV mtmv;
/**
* MaterializationContext, this contains necessary info for query rewriting by mv
*/
public AsyncMaterializationContext(MTMV mtmv, Plan mvPlan, Plan mvOriginalPlan, List<Table> baseTables,
List<Table> baseViews, CascadesContext cascadesContext) {
super(mvPlan, mvOriginalPlan, MaterializedViewUtils.generateMvScanPlan(mtmv, cascadesContext), cascadesContext);
this.mtmv = mtmv;
}
public MTMV getMtmv() {
return mtmv;
}
@Override
Plan doGenerateMvPlan(CascadesContext cascadesContext) {
return MaterializedViewUtils.generateMvScanPlan(this.mtmv, cascadesContext);
}
@Override
List<String> getMaterializationQualifier() {
return this.mtmv.getFullQualifiers();
}
@Override
String getStringInfo() {
StringBuilder failReasonBuilder = new StringBuilder("[").append("\n");
for (Map.Entry<ObjectId, Collection<Pair<String, String>>> reasonEntry : this.failReason.asMap().entrySet()) {
failReasonBuilder
.append("\n")
.append("ObjectId : ").append(reasonEntry.getKey()).append(".\n");
for (Pair<String, String> reason : reasonEntry.getValue()) {
failReasonBuilder.append("Summary : ").append(reason.key()).append(".\n")
.append("Reason : ").append(reason.value()).append(".\n");
}
}
failReasonBuilder.append("\n").append("]");
return Utils.toSqlString("MaterializationContext[" + getMaterializationQualifier() + "]",
"rewriteSuccess", this.success,
"failReason", failReasonBuilder.toString());
}
@Override
boolean isFinalChosen(Relation relation) {
if (!(relation instanceof PhysicalCatalogRelation)) {
return false;
}
return ((PhysicalCatalogRelation) relation).getTable() instanceof MTMV;
}
public Plan getMvScanPlan() {
return mvScanPlan;
}
public List<Table> getBaseTables() {
return baseTables;
}
public List<Table> getBaseViews() {
return baseViews;
}
public ExpressionMapping getMvExprToMvScanExprMapping() {
return mvExprToMvScanExprMapping;
}
public boolean isAvailable() {
return available;
}
public Plan getMvPlan() {
return mvPlan;
}
public Multimap<ObjectId, Pair<String, String>> getFailReason() {
return failReason;
}
public boolean isEnableRecordFailureDetail() {
return enableRecordFailureDetail;
}
public void setSuccess(boolean success) {
this.success = success;
this.failReason.clear();
}
public StructInfo getStructInfo() {
return structInfo;
}
public boolean isSuccess() {
return success;
}
}

19
fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/InitMaterializationContextHook.java
View File

@ -20,7 +20,9 @@ package org.apache.doris.nereids.rules.exploration.mv;
import org.apache.doris.catalog.Env;
import org.apache.doris.catalog.MTMV;
import org.apache.doris.catalog.TableIf;
import org.apache.doris.common.AnalysisException;
import org.apache.doris.mtmv.BaseTableInfo;
import org.apache.doris.mtmv.MTMVCache;
import org.apache.doris.nereids.CascadesContext;
import org.apache.doris.nereids.NereidsPlanner;
import org.apache.doris.nereids.PlannerHook;
@ -29,6 +31,7 @@ import org.apache.doris.nereids.trees.plans.visitor.TableCollector;
import org.apache.doris.nereids.trees.plans.visitor.TableCollector.TableCollectorContext;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.Sets;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
@ -75,10 +78,18 @@ public class InitMaterializationContextHook implements PlannerHook {
return;
}
for (MTMV materializedView : availableMTMVs) {
cascadesContext.addMaterializationContext(
MaterializationContext.fromMaterializedView(materializedView,
MaterializedViewUtils.generateMvScanPlan(materializedView, cascadesContext),
cascadesContext));
MTMVCache mtmvCache = null;
try {
mtmvCache = materializedView.getOrGenerateCache();
} catch (AnalysisException e) {
LOG.warn("MaterializationContext init mv cache generate fail", e);
}
if (mtmvCache == null) {
continue;
}
cascadesContext.addMaterializationContext(new AsyncMaterializationContext(materializedView,
mtmvCache.getLogicalPlan(), mtmvCache.getOriginalPlan(), ImmutableList.of(), ImmutableList.of(),
cascadesContext));
}
}
}

250
fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializationContext.java
View File

@ -18,102 +18,97 @@
package org.apache.doris.nereids.rules.exploration.mv;
import org.apache.doris.analysis.StatementBase;
import org.apache.doris.catalog.MTMV;
import org.apache.doris.catalog.Table;
import org.apache.doris.common.AnalysisException;
import org.apache.doris.common.Pair;
import org.apache.doris.mtmv.MTMVCache;
import org.apache.doris.nereids.CascadesContext;
import org.apache.doris.nereids.memo.GroupId;
import org.apache.doris.nereids.rules.exploration.mv.mapping.ExpressionMapping;
import org.apache.doris.nereids.trees.expressions.Expression;
import org.apache.doris.nereids.trees.plans.ObjectId;
import org.apache.doris.nereids.trees.plans.Plan;
import org.apache.doris.nereids.trees.plans.algebra.Relation;
import org.apache.doris.nereids.trees.plans.commands.ExplainCommand.ExplainLevel;
import org.apache.doris.nereids.trees.plans.physical.PhysicalPlan;
import org.apache.doris.nereids.trees.plans.physical.PhysicalRelation;
import org.apache.doris.nereids.trees.plans.visitor.DefaultPlanVisitor;
import org.apache.doris.nereids.util.ExpressionUtils;
import org.apache.doris.nereids.util.Utils;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.HashMultimap;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Multimap;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import java.util.BitSet;
import java.util.HashSet;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
import java.util.function.Supplier;
import java.util.stream.Collectors;
/**
* Maintain the context for query rewrite by materialized view
* Abstract context for query rewrite by materialized view
*/
public class MaterializationContext {
public abstract class MaterializationContext {
private static final Logger LOG = LogManager.getLogger(MaterializationContext.class);
private final MTMV mtmv;
private final List<Table> baseTables;
private final List<Table> baseViews;
// Group ids that are rewritten by this mv to reduce rewrite times
private final Set<GroupId> matchedFailGroups = new HashSet<>();
private final Set<GroupId> matchedSuccessGroups = new HashSet<>();
// if rewrite by mv fail, record the reason, if success the failReason should be empty.
// The key is the query belonged group expression objectId, the value is the fail reason
private final Map<ObjectId, Pair<String, String>> failReason = new LinkedHashMap<>();
protected List<Table> baseTables;
protected List<Table> baseViews;
// The plan of mv def sql
protected Plan mvPlan;
// The original plan of mv def sql
protected Plan originalMvPlan;
// Should regenerate when materialization is already rewritten successfully because one query may hit repeatedly
// make sure output is different in multi using
private Plan mvScanPlan;
// generated expressions form mv scan plan
private ExpressionMapping mvExprToMvScanExprMapping;
private List<? extends Expression> mvPlanOutputShuttledExpressions;
private boolean available = true;
// the mv plan from cache at present, record it to make sure query rewrite by mv is right when cache change.
private Plan mvPlan;
// mark rewrite success or not
private boolean success = false;
private boolean enableRecordFailureDetail = false;
private StructInfo structInfo;
protected Plan mvScanPlan;
// The mvPlan output shuttled expression, this is used by generate field mvExprToMvScanExprMapping
protected List<? extends Expression> mvPlanOutputShuttledExpressions;
// Generated mapping from mv plan out shuttled expr to mv scan plan out slot mapping, this is used for later used
protected ExpressionMapping mvExprToMvScanExprMapping;
// This mark the materialization context is available or not,
// will not be used in query transparent rewritten if false
protected boolean available = true;
// Mark the mv plan in the context is already rewritten successfully or not
protected boolean success = false;
// Mark enable record failure detail info or not, because record failure detail info is performance-depleting
protected final boolean enableRecordFailureDetail;
// The mv plan struct info
protected final StructInfo structInfo;
// Group id set that are rewritten unsuccessfully by this mv for reducing rewrite times
protected final Set<GroupId> matchedFailGroups = new HashSet<>();
// Group id set that are rewritten successfully by this mv for reducing rewrite times
protected final Set<GroupId> matchedSuccessGroups = new HashSet<>();
// Record the reason, if rewrite by mv fail. The failReason should be empty if success.
// The key is the query belonged group expression objectId, the value is the fail reasons because
// for one materialization query may be multi when nested materialized view.
protected final Multimap<ObjectId, Pair<String, String>> failReason = HashMultimap.create();
/**
* MaterializationContext, this contains necessary info for query rewriting by mv
*/
public MaterializationContext(MTMV mtmv, Plan mvScanPlan, List<Table> baseTables, List<Table> baseViews,
CascadesContext cascadesContext) {
this.mtmv = mtmv;
public MaterializationContext(Plan mvPlan, Plan originalMvPlan, Plan mvScanPlan, CascadesContext cascadesContext) {
this.mvPlan = mvPlan;
this.originalMvPlan = originalMvPlan;
this.mvScanPlan = mvScanPlan;
this.baseTables = baseTables;
this.baseViews = baseViews;
StatementBase parsedStatement = cascadesContext.getStatementContext().getParsedStatement();
this.enableRecordFailureDetail = parsedStatement != null && parsedStatement.isExplain()
&& ExplainLevel.MEMO_PLAN == parsedStatement.getExplainOptions().getExplainLevel();
MTMVCache mtmvCache = null;
try {
mtmvCache = mtmv.getOrGenerateCache();
} catch (AnalysisException e) {
LOG.warn("MaterializationContext init mv cache generate fail", e);
}
if (mtmvCache == null) {
this.available = false;
return;
}
this.mvPlanOutputShuttledExpressions = ExpressionUtils.shuttleExpressionWithLineage(
mtmvCache.getOriginalPlan().getOutput(),
mtmvCache.getOriginalPlan(),
originalMvPlan.getOutput(),
originalMvPlan,
new BitSet());
// mv output expression shuttle, this will be used to expression rewrite
this.mvExprToMvScanExprMapping = ExpressionMapping.generate(this.mvPlanOutputShuttledExpressions,
this.mvScanPlan.getExpressions());
this.mvScanPlan.getOutput());
// copy the plan from cache, which the plan in cache may change
this.mvPlan = mtmvCache.getLogicalPlan();
List<StructInfo> viewStructInfos = MaterializedViewUtils.extractStructInfo(
mtmvCache.getLogicalPlan(), cascadesContext, new BitSet());
mvPlan, cascadesContext, new BitSet());
if (viewStructInfos.size() > 1) {
// view struct info should only have one, log error and use the first struct info
LOG.warn(String.format("view strut info is more than one, mv name is %s, mv plan is %s",
mtmv.getName(), mvPlan.treeString()));
LOG.warn(String.format("view strut info is more than one, materialization name is %s, mv plan is %s",
getMaterializationQualifier(), getMvPlan().treeString()));
}
this.structInfo = viewStructInfos.get(0);
}
@ -131,22 +126,49 @@ public class MaterializationContext {
}
/**
* Try to generate scan plan for materialized view
* if MaterializationContext is already rewritten by materialized view, then should generate in real time
* when query rewrite, because one plan may hit the materialized view repeatedly and the mv scan output
* Try to generate scan plan for materialization
* if MaterializationContext is already rewritten successfully, then should generate new scan plan in later
* query rewrite, because one plan may hit the materialized view repeatedly and the mv scan output
* should be different
*/
public void tryReGenerateMvScanPlan(CascadesContext cascadesContext) {
if (!this.matchedSuccessGroups.isEmpty()) {
this.mvScanPlan = MaterializedViewUtils.generateMvScanPlan(this.mtmv, cascadesContext);
this.mvScanPlan = doGenerateMvPlan(cascadesContext);
// mv output expression shuttle, this will be used to expression rewrite
this.mvExprToMvScanExprMapping = ExpressionMapping.generate(this.mvPlanOutputShuttledExpressions,
this.mvScanPlan.getExpressions());
}
}
public MTMV getMTMV() {
return mtmv;
/**
* Try to generate scan plan for materialization
* if MaterializationContext is already rewritten successfully, then should generate new scan plan in later
* query rewrite, because one plan may hit the materialized view repeatedly and the mv scan output
* should be different
*/
abstract Plan doGenerateMvPlan(CascadesContext cascadesContext);
/**
* Get materialization unique qualifier which identify it
*/
abstract List<String> getMaterializationQualifier();
/**
* Get String info which is used for to string
*/
abstract String getStringInfo();
/**
* Calc the relation is chosen finally or not
*/
abstract boolean isFinalChosen(Relation relation);
public Plan getMvPlan() {
return mvPlan;
}
public Plan getOriginalMvPlan() {
return originalMvPlan;
}
public Plan getMvScanPlan() {
@ -169,11 +191,7 @@ public class MaterializationContext {
return available;
}
public Plan getMvPlan() {
return mvPlan;
}
public Map<ObjectId, Pair<String, String>> getFailReason() {
public Multimap<ObjectId, Pair<String, String>> getFailReason() {
return failReason;
}
@ -183,6 +201,7 @@ public class MaterializationContext {
public void setSuccess(boolean success) {
this.success = success;
// TODO clear the fail message by according planId ?
this.failReason.clear();
}
@ -190,8 +209,12 @@ public class MaterializationContext {
return structInfo;
}
public boolean isSuccess() {
return success;
}
/**
* recordFailReason
* Record fail reason when in rewriting
*/
public void recordFailReason(StructInfo structInfo, String summary, Supplier<String> failureReasonSupplier) {
// record it's rewritten
@ -207,65 +230,54 @@ public class MaterializationContext {
Pair.of(summary, this.isEnableRecordFailureDetail() ? failureReasonSupplier.get() : ""));
}
public boolean isSuccess() {
return success;
}
@Override
public String toString() {
StringBuilder failReasonBuilder = new StringBuilder("[").append("\n");
for (Map.Entry<ObjectId, Pair<String, String>> reason : this.failReason.entrySet()) {
failReasonBuilder
.append("\n")
.append("ObjectId : ").append(reason.getKey()).append(".\n")
.append("Summary : ").append(reason.getValue().key()).append(".\n")
.append("Reason : ").append(reason.getValue().value()).append(".\n");
}
failReasonBuilder.append("\n").append("]");
return Utils.toSqlString("MaterializationContext[" + mtmv.getName() + "]",
"rewriteSuccess", this.success,
"failReason", failReasonBuilder.toString());
return getStringInfo();
}
/**
* toString, this contains summary and detail info.
*/
public static String toDetailString(List<MaterializationContext> materializationContexts) {
StringBuilder builder = new StringBuilder();
builder.append("materializationContexts:").append("\n");
for (MaterializationContext ctx : materializationContexts) {
builder.append("\n").append(ctx).append("\n");
}
return builder.toString();
}
/**
* toSummaryString, this contains only summary info.
* ToSummaryString, this contains only summary info.
*/
public static String toSummaryString(List<MaterializationContext> materializationContexts,
List<MTMV> chosenMaterializationNames) {
PhysicalPlan physicalPlan) {
if (materializationContexts.isEmpty()) {
return "";
}
Set<String> materializationChosenNameSet = chosenMaterializationNames.stream()
.map(MTMV::getName)
Set<MaterializationContext> rewrittenSuccessMaterializationSet = materializationContexts.stream()
.filter(MaterializationContext::isSuccess)
.collect(Collectors.toSet());
Set<List<String>> chosenMaterializationQualifiers = new HashSet<>();
physicalPlan.accept(new DefaultPlanVisitor<Void, Void>() {
@Override
public Void visitPhysicalRelation(PhysicalRelation physicalRelation, Void context) {
for (MaterializationContext rewrittenContext : rewrittenSuccessMaterializationSet) {
if (rewrittenContext.isFinalChosen(physicalRelation)) {
chosenMaterializationQualifiers.add(rewrittenContext.getMaterializationQualifier());
}
}
return null;
}
}, null);
StringBuilder builder = new StringBuilder();
builder.append("\nMaterializedView");
// rewrite success and chosen
builder.append("\nMaterializedViewRewriteSuccessAndChose:\n");
if (!materializationChosenNameSet.isEmpty()) {
builder.append(" Names: ").append(String.join(", ", materializationChosenNameSet));
if (!chosenMaterializationQualifiers.isEmpty()) {
builder.append(" Names: ");
chosenMaterializationQualifiers.forEach(materializationQualifier ->
builder.append(generateQualifierName(materializationQualifier)).append(", "));
}
// rewrite success but not chosen
builder.append("\nMaterializedViewRewriteSuccessButNotChose:\n");
Set<String> rewriteSuccessButNotChoseNameSet = materializationContexts.stream()
.filter(materializationContext -> materializationContext.isSuccess()
&& !materializationChosenNameSet.contains(materializationContext.getMTMV().getName()))
.map(materializationContext -> materializationContext.getMTMV().getName())
Set<List<String>> rewriteSuccessButNotChoseQualifiers = rewrittenSuccessMaterializationSet.stream()
.map(MaterializationContext::getMaterializationQualifier)
.filter(materializationQualifier -> !chosenMaterializationQualifiers.contains(materializationQualifier))
.collect(Collectors.toSet());
if (!rewriteSuccessButNotChoseNameSet.isEmpty()) {
builder.append(" Names: ").append(String.join(", ", rewriteSuccessButNotChoseNameSet));
if (!rewriteSuccessButNotChoseQualifiers.isEmpty()) {
builder.append(" Names: ");
rewriteSuccessButNotChoseQualifiers.forEach(materializationQualifier ->
builder.append(generateQualifierName(materializationQualifier)).append(", "));
}
// rewrite fail
builder.append("\nMaterializedViewRewriteFail:");
@ -274,7 +286,7 @@ public class MaterializationContext {
Set<String> failReasonSet =
ctx.getFailReason().values().stream().map(Pair::key).collect(ImmutableSet.toImmutableSet());
builder.append("\n")
.append(" Name: ").append(ctx.getMTMV().getName())
.append(" Name: ").append(generateQualifierName(ctx.getMaterializationQualifier()))
.append("\n")
.append(" FailSummary: ").append(String.join(", ", failReasonSet));
}
@ -282,12 +294,24 @@ public class MaterializationContext {
return builder.toString();
}
/**
* MaterializationContext fromMaterializedView
*/
public static MaterializationContext fromMaterializedView(MTMV materializedView, Plan mvScanPlan,
CascadesContext cascadesContext) {
return new MaterializationContext(materializedView, mvScanPlan, ImmutableList.of(), ImmutableList.of(),
cascadesContext);
private static String generateQualifierName(List<String> qualifiers) {
return String.join("#", qualifiers);
}
@Override
public boolean equals(Object o) {
if (this == o) {
return true;
}
if (o == null || getClass() != o.getClass()) {
return false;
}
MaterializationContext context = (MaterializationContext) o;
return getMaterializationQualifier().equals(context.getMaterializationQualifier());
}
@Override
public int hashCode() {
return Objects.hash(getMaterializationQualifier());
}
}

3
fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewUtils.java
View File

@ -203,6 +203,9 @@ public class MaterializedViewUtils {
CascadesContext cascadesContext,
Function<CascadesContext, Plan> planRewriter,
Plan rewrittenPlan, Plan originPlan) {
if (originPlan == null || rewrittenPlan == null) {
return null;
}
if (originPlan.getOutputSet().size() != rewrittenPlan.getOutputSet().size()) {
return rewrittenPlan;
}

7
fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/StructInfo.java
View File

@ -17,7 +17,6 @@
package org.apache.doris.nereids.rules.exploration.mv;
import org.apache.doris.catalog.MTMV;
import org.apache.doris.catalog.PartitionInfo;
import org.apache.doris.catalog.PartitionItem;
import org.apache.doris.catalog.TableIf;
@ -630,11 +629,11 @@ public class StructInfo {
/**
* Add predicates on base table when materialized view scan contains invalid partitions
*/
public static class InvalidPartitionRemover extends DefaultPlanRewriter<Pair<MTMV, Set<Long>>> {
public static class InvalidPartitionRemover extends DefaultPlanRewriter<Pair<List<String>, Set<Long>>> {
// materialized view scan is always LogicalOlapScan, so just handle LogicalOlapScan
@Override
public Plan visitLogicalOlapScan(LogicalOlapScan olapScan, Pair<MTMV, Set<Long>> context) {
if (olapScan.getTable().getName().equals(context.key().getName())) {
public Plan visitLogicalOlapScan(LogicalOlapScan olapScan, Pair<List<String>, Set<Long>> context) {
if (olapScan.getTable().getFullQualifiers().equals(context.key())) {
List<Long> selectedPartitionIds = olapScan.getSelectedPartitionIds();
return olapScan.withSelectedPartitionIds(selectedPartitionIds.stream()
.filter(partitionId -> !context.value().contains(partitionId))