diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewRule.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewRule.java index 41157a6c86..a88362e669 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewRule.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewRule.java @@ -179,7 +179,12 @@ public abstract class AbstractMaterializedViewRule implements ExplorationRuleFac return rewriteResults; } for (RelationMapping queryToViewTableMapping : queryToViewTableMappings) { - SlotMapping queryToViewSlotMapping = SlotMapping.generate(queryToViewTableMapping); + SlotMapping queryToViewSlotMapping = + materializationContext.getSlotMappingFromCache(queryToViewTableMapping); + if (queryToViewSlotMapping == null) { + queryToViewSlotMapping = SlotMapping.generate(queryToViewTableMapping); + materializationContext.addSlotMappingToCache(queryToViewTableMapping, queryToViewSlotMapping); + } if (queryToViewSlotMapping == null) { materializationContext.recordFailReason(queryStructInfo, "Query to view slot mapping is null", () -> ""); @@ -187,7 +192,7 @@ public abstract class AbstractMaterializedViewRule implements ExplorationRuleFac } SlotMapping viewToQuerySlotMapping = queryToViewSlotMapping.inverse(); LogicalCompatibilityContext compatibilityContext = LogicalCompatibilityContext.from( - queryToViewTableMapping, queryToViewSlotMapping, queryStructInfo, viewStructInfo); + queryToViewTableMapping, viewToQuerySlotMapping, queryStructInfo, viewStructInfo); ComparisonResult comparisonResult = StructInfo.isGraphLogicalEquals(queryStructInfo, viewStructInfo, compatibilityContext); if (comparisonResult.isInvalid()) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/LogicalCompatibilityContext.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/LogicalCompatibilityContext.java index b4ed509f30..25bafeb64c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/LogicalCompatibilityContext.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/LogicalCompatibilityContext.java @@ -128,7 +128,7 @@ public class LogicalCompatibilityContext { * this make expression mapping between query and view by relation and the slot in relation mapping */ public static LogicalCompatibilityContext from(RelationMapping relationMapping, - SlotMapping queryToViewSlotMapping, + SlotMapping viewToQuerySlotMapping, StructInfo queryStructInfo, StructInfo viewStructInfo) { // init node mapping @@ -147,11 +147,8 @@ public class LogicalCompatibilityContext { queryToViewNodeMapping.put(queryStructInfoNode, viewStructInfoNode); } } - // init expression mapping - Map viewToQuerySlotMapping = queryToViewSlotMapping.inverse() - .toSlotReferenceMap(); return new LogicalCompatibilityContext(queryToViewNodeMapping, - viewToQuerySlotMapping, + viewToQuerySlotMapping.toSlotReferenceMap(), queryStructInfo, viewStructInfo); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializationContext.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializationContext.java index 261e3bb85f..2f0d04e114 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializationContext.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializationContext.java @@ -23,6 +23,8 @@ import org.apache.doris.common.Pair; import org.apache.doris.nereids.CascadesContext; import org.apache.doris.nereids.memo.GroupId; import org.apache.doris.nereids.rules.exploration.mv.mapping.ExpressionMapping; +import org.apache.doris.nereids.rules.exploration.mv.mapping.RelationMapping; +import org.apache.doris.nereids.rules.exploration.mv.mapping.SlotMapping; import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.plans.ObjectId; import org.apache.doris.nereids.trees.plans.Plan; @@ -40,8 +42,10 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import java.util.BitSet; +import java.util.HashMap; import java.util.HashSet; import java.util.List; +import java.util.Map; import java.util.Objects; import java.util.Set; import java.util.function.Supplier; @@ -52,6 +56,7 @@ import java.util.stream.Collectors; */ public abstract class MaterializationContext { private static final Logger LOG = LogManager.getLogger(MaterializationContext.class); + public final Map queryToMvSlotMappingCache = new HashMap<>(); protected List baseTables; protected List
baseViews; // The plan of mv def sql @@ -140,6 +145,14 @@ public abstract class MaterializationContext { } } + public void addSlotMappingToCache(RelationMapping relationMapping, SlotMapping slotMapping) { + queryToMvSlotMappingCache.put(relationMapping, slotMapping); + } + + public SlotMapping getSlotMappingFromCache(RelationMapping relationMapping) { + return queryToMvSlotMappingCache.get(relationMapping); + } + /** * Try to generate scan plan for materialization * if MaterializationContext is already rewritten successfully, then should generate new scan plan in later diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/mapping/ExpressionMapping.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/mapping/ExpressionMapping.java index 5a5bfedfe1..8c77eacfaf 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/mapping/ExpressionMapping.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/mapping/ExpressionMapping.java @@ -23,7 +23,6 @@ import org.apache.doris.nereids.util.ExpressionUtils; import org.apache.doris.nereids.util.Utils; import com.google.common.collect.ArrayListMultimap; -import com.google.common.collect.ImmutableMultimap; import com.google.common.collect.Lists; import com.google.common.collect.Multimap; @@ -32,7 +31,6 @@ import java.util.Collection; import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.Map.Entry; /** * Expression mapping, maybe one expression map to multi expression @@ -104,27 +102,6 @@ public class ExpressionMapping extends Mapping { return new ExpressionMapping(expressionMultiMap); } - @Override - public Mapping chainedFold(Mapping target) { - - ImmutableMultimap.Builder foldedMappingBuilder = - ImmutableMultimap.builder(); - - Multimap targetMapping - = ((ExpressionMapping) target).getExpressionMapping(); - for (Entry> exprMapping : - this.getExpressionMapping().asMap().entrySet()) { - Collection valueExpressions = exprMapping.getValue(); - valueExpressions.forEach(valueExpr -> { - if (targetMapping.containsKey(valueExpr)) { - targetMapping.get(valueExpr).forEach( - targetValue -> foldedMappingBuilder.put(exprMapping.getKey(), targetValue)); - } - }); - } - return new ExpressionMapping(foldedMappingBuilder.build()); - } - @Override public String toString() { return Utils.toSqlString("ExpressionMapping", "expressionMapping", expressionMapping); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/mapping/Mapping.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/mapping/Mapping.java index 18fa282267..e14b79a52e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/mapping/Mapping.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/mapping/Mapping.java @@ -22,6 +22,8 @@ import org.apache.doris.nereids.trees.expressions.Slot; import org.apache.doris.nereids.trees.plans.RelationId; import org.apache.doris.nereids.trees.plans.algebra.CatalogRelation; +import java.util.HashMap; +import java.util.Map; import java.util.Objects; import javax.annotation.Nullable; @@ -38,10 +40,15 @@ public abstract class Mapping { public final RelationId relationId; public final CatalogRelation belongedRelation; + // Generate eagerly, will be used to generate slot mapping + private final Map slotNameToSlotMap = new HashMap<>(); public MappedRelation(RelationId relationId, CatalogRelation belongedRelation) { this.relationId = relationId; this.belongedRelation = belongedRelation; + for (Slot slot : belongedRelation.getOutput()) { + slotNameToSlotMap.put(slot.getName(), slot); + } } public static MappedRelation of(RelationId relationId, CatalogRelation belongedRelation) { @@ -56,6 +63,10 @@ public abstract class Mapping { return belongedRelation; } + public Map getSlotNameToSlotMap() { + return slotNameToSlotMap; + } + @Override public boolean equals(Object o) { if (this == o) { @@ -72,6 +83,11 @@ public abstract class Mapping { public int hashCode() { return Objects.hash(relationId); } + + @Override + public String toString() { + return "MappedRelation{" + "relationId=" + relationId + ", slotNameToSlotMap=" + slotNameToSlotMap + '}'; + } } /** @@ -141,12 +157,4 @@ public abstract class Mapping { return "MappedSlot{" + "slot=" + slot + '}'; } } - - /** Chain fold tow mapping, such as this mapping is {[a -> b]}, the target mapping is - * {[b -> c]} after chain fold, this result will be {[a -> c]}, if the value side in this mapping - * can get the key in the target mapping, will lose the mapping - */ - protected Mapping chainedFold(Mapping target) { - return null; - } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/mapping/RelationMapping.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/mapping/RelationMapping.java index b5494c01b1..eb53923da5 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/mapping/RelationMapping.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/mapping/RelationMapping.java @@ -32,6 +32,7 @@ import com.google.common.collect.Sets; import java.util.ArrayList; import java.util.List; +import java.util.Objects; import java.util.Set; /** @@ -143,4 +144,21 @@ public class RelationMapping extends Mapping { private static Long getTableQualifier(TableIf tableIf) { return tableIf.getId(); } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + RelationMapping that = (RelationMapping) o; + return Objects.equals(mappedRelationMap, that.mappedRelationMap); + } + + @Override + public int hashCode() { + return Objects.hash(mappedRelationMap); + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/mapping/SlotMapping.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/mapping/SlotMapping.java index f95bcedd2f..8384b3e094 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/mapping/SlotMapping.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/mapping/SlotMapping.java @@ -22,10 +22,11 @@ import org.apache.doris.nereids.trees.expressions.SlotReference; import com.google.common.collect.BiMap; import com.google.common.collect.HashBiMap; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import java.util.HashMap; import java.util.Map; -import java.util.stream.Collectors; import javax.annotation.Nullable; /** @@ -33,11 +34,15 @@ import javax.annotation.Nullable; */ public class SlotMapping extends Mapping { + public static final Logger LOG = LogManager.getLogger(SlotMapping.class); + private final BiMap relationSlotMap; private Map slotReferenceMap; - public SlotMapping(BiMap relationSlotMap) { + public SlotMapping(BiMap relationSlotMap, + Map slotReferenceMap) { this.relationSlotMap = relationSlotMap; + this.slotReferenceMap = slotReferenceMap; } public BiMap getRelationSlotMap() { @@ -45,11 +50,12 @@ public class SlotMapping extends Mapping { } public SlotMapping inverse() { - return SlotMapping.of(relationSlotMap.inverse()); + return SlotMapping.of(relationSlotMap.inverse(), null); } - public static SlotMapping of(BiMap relationSlotMap) { - return new SlotMapping(relationSlotMap); + public static SlotMapping of(BiMap relationSlotMap, + Map slotReferenceMap) { + return new SlotMapping(relationSlotMap, slotReferenceMap); } /** @@ -58,26 +64,31 @@ public class SlotMapping extends Mapping { @Nullable public static SlotMapping generate(RelationMapping relationMapping) { BiMap relationSlotMap = HashBiMap.create(); + Map slotReferenceMap = new HashMap<>(); BiMap mappedRelationMap = relationMapping.getMappedRelationMap(); for (Map.Entry mappedRelationEntry : mappedRelationMap.entrySet()) { - Map targetNameSlotMap = - mappedRelationEntry.getValue().getBelongedRelation().getOutput().stream() - .collect(Collectors.toMap(Slot::getName, slot -> slot)); - for (Slot sourceSlot : mappedRelationEntry.getKey().getBelongedRelation().getOutput()) { - Slot targetSlot = targetNameSlotMap.get(sourceSlot.getName()); + MappedRelation sourceRelation = mappedRelationEntry.getKey(); + Map sourceSlotNameToSlotMap = sourceRelation.getSlotNameToSlotMap(); + + MappedRelation targetRelation = mappedRelationEntry.getValue(); + Map targetSlotNameSlotMap = targetRelation.getSlotNameToSlotMap(); + + for (String sourceSlotName : sourceSlotNameToSlotMap.keySet()) { + Slot targetSlot = targetSlotNameSlotMap.get(sourceSlotName); // source slot can not map from target, bail out if (targetSlot == null) { + LOG.warn(String.format("SlotMapping generate is null, source relation is %s, " + + "target relation is %s", sourceRelation, targetRelation)); return null; } - relationSlotMap.put(MappedSlot.of(sourceSlot, mappedRelationEntry.getKey().getBelongedRelation()), - MappedSlot.of(targetSlot, mappedRelationEntry.getValue().getBelongedRelation())); + Slot sourceSlot = sourceSlotNameToSlotMap.get(sourceSlotName); + relationSlotMap.put(MappedSlot.of(sourceSlot, + sourceRelation.getBelongedRelation()), + MappedSlot.of(targetSlot, targetRelation.getBelongedRelation())); + slotReferenceMap.put((SlotReference) sourceSlot, (SlotReference) targetSlot); } } - return SlotMapping.of(relationSlotMap); - } - - public Map toMappedSlotMap() { - return (Map) this.getRelationSlotMap(); + return SlotMapping.of(relationSlotMap, slotReferenceMap); } /** @@ -87,12 +98,11 @@ public class SlotMapping extends Mapping { if (this.slotReferenceMap != null) { return this.slotReferenceMap; } - Map slotReferenceSlotReferenceMap = new HashMap<>(); + this.slotReferenceMap = new HashMap<>(); for (Map.Entry entry : this.getRelationSlotMap().entrySet()) { - slotReferenceSlotReferenceMap.put((SlotReference) entry.getKey().getSlot(), + this.slotReferenceMap.put((SlotReference) entry.getKey().getSlot(), (SlotReference) entry.getValue().getSlot()); } - this.slotReferenceMap = slotReferenceSlotReferenceMap; return this.slotReferenceMap; }