From 71caf88ec1563e01329fa0202f2a7cf854d94f6d Mon Sep 17 00:00:00 2001 From: seawinde <149132972+seawinde@users.noreply.github.com> Date: Wed, 15 May 2024 13:57:28 +0800 Subject: [PATCH] [opt](mtmv) Optimize the logic of slot mapping generate for performance (#34597) Slot mapping is used for materialized view rewritting given the relation mapping, the slot mapping is the same Optimize the slot mapping genarate logic Cache the slot mapping in materialization context by realation mapping key --- .../mv/AbstractMaterializedViewRule.java | 9 +++- .../mv/LogicalCompatibilityContext.java | 7 +-- .../mv/MaterializationContext.java | 13 +++++ .../mv/mapping/ExpressionMapping.java | 23 --------- .../rules/exploration/mv/mapping/Mapping.java | 24 ++++++--- .../mv/mapping/RelationMapping.java | 18 +++++++ .../exploration/mv/mapping/SlotMapping.java | 50 +++++++++++-------- 7 files changed, 86 insertions(+), 58 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewRule.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewRule.java index 41157a6c86..a88362e669 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewRule.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewRule.java @@ -179,7 +179,12 @@ public abstract class AbstractMaterializedViewRule implements ExplorationRuleFac return rewriteResults; } for (RelationMapping queryToViewTableMapping : queryToViewTableMappings) { - SlotMapping queryToViewSlotMapping = SlotMapping.generate(queryToViewTableMapping); + SlotMapping queryToViewSlotMapping = + materializationContext.getSlotMappingFromCache(queryToViewTableMapping); + if (queryToViewSlotMapping == null) { + queryToViewSlotMapping = SlotMapping.generate(queryToViewTableMapping); + materializationContext.addSlotMappingToCache(queryToViewTableMapping, queryToViewSlotMapping); + } if (queryToViewSlotMapping == null) { materializationContext.recordFailReason(queryStructInfo, "Query to view slot mapping is null", () -> ""); @@ -187,7 +192,7 @@ public abstract class AbstractMaterializedViewRule implements ExplorationRuleFac } SlotMapping viewToQuerySlotMapping = queryToViewSlotMapping.inverse(); LogicalCompatibilityContext compatibilityContext = LogicalCompatibilityContext.from( - queryToViewTableMapping, queryToViewSlotMapping, queryStructInfo, viewStructInfo); + queryToViewTableMapping, viewToQuerySlotMapping, queryStructInfo, viewStructInfo); ComparisonResult comparisonResult = StructInfo.isGraphLogicalEquals(queryStructInfo, viewStructInfo, compatibilityContext); if (comparisonResult.isInvalid()) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/LogicalCompatibilityContext.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/LogicalCompatibilityContext.java index b4ed509f30..25bafeb64c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/LogicalCompatibilityContext.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/LogicalCompatibilityContext.java @@ -128,7 +128,7 @@ public class LogicalCompatibilityContext { * this make expression mapping between query and view by relation and the slot in relation mapping */ public static LogicalCompatibilityContext from(RelationMapping relationMapping, - SlotMapping queryToViewSlotMapping, + SlotMapping viewToQuerySlotMapping, StructInfo queryStructInfo, StructInfo viewStructInfo) { // init node mapping @@ -147,11 +147,8 @@ public class LogicalCompatibilityContext { queryToViewNodeMapping.put(queryStructInfoNode, viewStructInfoNode); } } - // init expression mapping - Map viewToQuerySlotMapping = queryToViewSlotMapping.inverse() - .toSlotReferenceMap(); return new LogicalCompatibilityContext(queryToViewNodeMapping, - viewToQuerySlotMapping, + viewToQuerySlotMapping.toSlotReferenceMap(), queryStructInfo, viewStructInfo); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializationContext.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializationContext.java index 261e3bb85f..2f0d04e114 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializationContext.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializationContext.java @@ -23,6 +23,8 @@ import org.apache.doris.common.Pair; import org.apache.doris.nereids.CascadesContext; import org.apache.doris.nereids.memo.GroupId; import org.apache.doris.nereids.rules.exploration.mv.mapping.ExpressionMapping; +import org.apache.doris.nereids.rules.exploration.mv.mapping.RelationMapping; +import org.apache.doris.nereids.rules.exploration.mv.mapping.SlotMapping; import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.plans.ObjectId; import org.apache.doris.nereids.trees.plans.Plan; @@ -40,8 +42,10 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import java.util.BitSet; +import java.util.HashMap; import java.util.HashSet; import java.util.List; +import java.util.Map; import java.util.Objects; import java.util.Set; import java.util.function.Supplier; @@ -52,6 +56,7 @@ import java.util.stream.Collectors; */ public abstract class MaterializationContext { private static final Logger LOG = LogManager.getLogger(MaterializationContext.class); + public final Map queryToMvSlotMappingCache = new HashMap<>(); protected List baseTables; protected List
baseViews; // The plan of mv def sql @@ -140,6 +145,14 @@ public abstract class MaterializationContext { } } + public void addSlotMappingToCache(RelationMapping relationMapping, SlotMapping slotMapping) { + queryToMvSlotMappingCache.put(relationMapping, slotMapping); + } + + public SlotMapping getSlotMappingFromCache(RelationMapping relationMapping) { + return queryToMvSlotMappingCache.get(relationMapping); + } + /** * Try to generate scan plan for materialization * if MaterializationContext is already rewritten successfully, then should generate new scan plan in later diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/mapping/ExpressionMapping.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/mapping/ExpressionMapping.java index 5a5bfedfe1..8c77eacfaf 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/mapping/ExpressionMapping.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/mapping/ExpressionMapping.java @@ -23,7 +23,6 @@ import org.apache.doris.nereids.util.ExpressionUtils; import org.apache.doris.nereids.util.Utils; import com.google.common.collect.ArrayListMultimap; -import com.google.common.collect.ImmutableMultimap; import com.google.common.collect.Lists; import com.google.common.collect.Multimap; @@ -32,7 +31,6 @@ import java.util.Collection; import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.Map.Entry; /** * Expression mapping, maybe one expression map to multi expression @@ -104,27 +102,6 @@ public class ExpressionMapping extends Mapping { return new ExpressionMapping(expressionMultiMap); } - @Override - public Mapping chainedFold(Mapping target) { - - ImmutableMultimap.Builder foldedMappingBuilder = - ImmutableMultimap.builder(); - - Multimap targetMapping - = ((ExpressionMapping) target).getExpressionMapping(); - for (Entry> exprMapping : - this.getExpressionMapping().asMap().entrySet()) { - Collection valueExpressions = exprMapping.getValue(); - valueExpressions.forEach(valueExpr -> { - if (targetMapping.containsKey(valueExpr)) { - targetMapping.get(valueExpr).forEach( - targetValue -> foldedMappingBuilder.put(exprMapping.getKey(), targetValue)); - } - }); - } - return new ExpressionMapping(foldedMappingBuilder.build()); - } - @Override public String toString() { return Utils.toSqlString("ExpressionMapping", "expressionMapping", expressionMapping); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/mapping/Mapping.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/mapping/Mapping.java index 18fa282267..e14b79a52e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/mapping/Mapping.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/mapping/Mapping.java @@ -22,6 +22,8 @@ import org.apache.doris.nereids.trees.expressions.Slot; import org.apache.doris.nereids.trees.plans.RelationId; import org.apache.doris.nereids.trees.plans.algebra.CatalogRelation; +import java.util.HashMap; +import java.util.Map; import java.util.Objects; import javax.annotation.Nullable; @@ -38,10 +40,15 @@ public abstract class Mapping { public final RelationId relationId; public final CatalogRelation belongedRelation; + // Generate eagerly, will be used to generate slot mapping + private final Map slotNameToSlotMap = new HashMap<>(); public MappedRelation(RelationId relationId, CatalogRelation belongedRelation) { this.relationId = relationId; this.belongedRelation = belongedRelation; + for (Slot slot : belongedRelation.getOutput()) { + slotNameToSlotMap.put(slot.getName(), slot); + } } public static MappedRelation of(RelationId relationId, CatalogRelation belongedRelation) { @@ -56,6 +63,10 @@ public abstract class Mapping { return belongedRelation; } + public Map getSlotNameToSlotMap() { + return slotNameToSlotMap; + } + @Override public boolean equals(Object o) { if (this == o) { @@ -72,6 +83,11 @@ public abstract class Mapping { public int hashCode() { return Objects.hash(relationId); } + + @Override + public String toString() { + return "MappedRelation{" + "relationId=" + relationId + ", slotNameToSlotMap=" + slotNameToSlotMap + '}'; + } } /** @@ -141,12 +157,4 @@ public abstract class Mapping { return "MappedSlot{" + "slot=" + slot + '}'; } } - - /** Chain fold tow mapping, such as this mapping is {[a -> b]}, the target mapping is - * {[b -> c]} after chain fold, this result will be {[a -> c]}, if the value side in this mapping - * can get the key in the target mapping, will lose the mapping - */ - protected Mapping chainedFold(Mapping target) { - return null; - } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/mapping/RelationMapping.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/mapping/RelationMapping.java index b5494c01b1..eb53923da5 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/mapping/RelationMapping.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/mapping/RelationMapping.java @@ -32,6 +32,7 @@ import com.google.common.collect.Sets; import java.util.ArrayList; import java.util.List; +import java.util.Objects; import java.util.Set; /** @@ -143,4 +144,21 @@ public class RelationMapping extends Mapping { private static Long getTableQualifier(TableIf tableIf) { return tableIf.getId(); } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + RelationMapping that = (RelationMapping) o; + return Objects.equals(mappedRelationMap, that.mappedRelationMap); + } + + @Override + public int hashCode() { + return Objects.hash(mappedRelationMap); + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/mapping/SlotMapping.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/mapping/SlotMapping.java index f95bcedd2f..8384b3e094 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/mapping/SlotMapping.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/mapping/SlotMapping.java @@ -22,10 +22,11 @@ import org.apache.doris.nereids.trees.expressions.SlotReference; import com.google.common.collect.BiMap; import com.google.common.collect.HashBiMap; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import java.util.HashMap; import java.util.Map; -import java.util.stream.Collectors; import javax.annotation.Nullable; /** @@ -33,11 +34,15 @@ import javax.annotation.Nullable; */ public class SlotMapping extends Mapping { + public static final Logger LOG = LogManager.getLogger(SlotMapping.class); + private final BiMap relationSlotMap; private Map slotReferenceMap; - public SlotMapping(BiMap relationSlotMap) { + public SlotMapping(BiMap relationSlotMap, + Map slotReferenceMap) { this.relationSlotMap = relationSlotMap; + this.slotReferenceMap = slotReferenceMap; } public BiMap getRelationSlotMap() { @@ -45,11 +50,12 @@ public class SlotMapping extends Mapping { } public SlotMapping inverse() { - return SlotMapping.of(relationSlotMap.inverse()); + return SlotMapping.of(relationSlotMap.inverse(), null); } - public static SlotMapping of(BiMap relationSlotMap) { - return new SlotMapping(relationSlotMap); + public static SlotMapping of(BiMap relationSlotMap, + Map slotReferenceMap) { + return new SlotMapping(relationSlotMap, slotReferenceMap); } /** @@ -58,26 +64,31 @@ public class SlotMapping extends Mapping { @Nullable public static SlotMapping generate(RelationMapping relationMapping) { BiMap relationSlotMap = HashBiMap.create(); + Map slotReferenceMap = new HashMap<>(); BiMap mappedRelationMap = relationMapping.getMappedRelationMap(); for (Map.Entry mappedRelationEntry : mappedRelationMap.entrySet()) { - Map targetNameSlotMap = - mappedRelationEntry.getValue().getBelongedRelation().getOutput().stream() - .collect(Collectors.toMap(Slot::getName, slot -> slot)); - for (Slot sourceSlot : mappedRelationEntry.getKey().getBelongedRelation().getOutput()) { - Slot targetSlot = targetNameSlotMap.get(sourceSlot.getName()); + MappedRelation sourceRelation = mappedRelationEntry.getKey(); + Map sourceSlotNameToSlotMap = sourceRelation.getSlotNameToSlotMap(); + + MappedRelation targetRelation = mappedRelationEntry.getValue(); + Map targetSlotNameSlotMap = targetRelation.getSlotNameToSlotMap(); + + for (String sourceSlotName : sourceSlotNameToSlotMap.keySet()) { + Slot targetSlot = targetSlotNameSlotMap.get(sourceSlotName); // source slot can not map from target, bail out if (targetSlot == null) { + LOG.warn(String.format("SlotMapping generate is null, source relation is %s, " + + "target relation is %s", sourceRelation, targetRelation)); return null; } - relationSlotMap.put(MappedSlot.of(sourceSlot, mappedRelationEntry.getKey().getBelongedRelation()), - MappedSlot.of(targetSlot, mappedRelationEntry.getValue().getBelongedRelation())); + Slot sourceSlot = sourceSlotNameToSlotMap.get(sourceSlotName); + relationSlotMap.put(MappedSlot.of(sourceSlot, + sourceRelation.getBelongedRelation()), + MappedSlot.of(targetSlot, targetRelation.getBelongedRelation())); + slotReferenceMap.put((SlotReference) sourceSlot, (SlotReference) targetSlot); } } - return SlotMapping.of(relationSlotMap); - } - - public Map toMappedSlotMap() { - return (Map) this.getRelationSlotMap(); + return SlotMapping.of(relationSlotMap, slotReferenceMap); } /** @@ -87,12 +98,11 @@ public class SlotMapping extends Mapping { if (this.slotReferenceMap != null) { return this.slotReferenceMap; } - Map slotReferenceSlotReferenceMap = new HashMap<>(); + this.slotReferenceMap = new HashMap<>(); for (Map.Entry entry : this.getRelationSlotMap().entrySet()) { - slotReferenceSlotReferenceMap.put((SlotReference) entry.getKey().getSlot(), + this.slotReferenceMap.put((SlotReference) entry.getKey().getSlot(), (SlotReference) entry.getValue().getSlot()); } - this.slotReferenceMap = slotReferenceSlotReferenceMap; return this.slotReferenceMap; }