database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[opt](mtmv) Optimize the logic of slot mapping generate for performance (#34597)

Browse Source 
Slot mapping is used for materialized view rewritting
given the relation mapping, the slot mapping is the same

Optimize the slot mapping genarate logic
Cache the slot mapping in materialization context by realation mapping key
This commit is contained in:
seawinde
2024-05-15 13:57:28 +08:00
committed by yiguolei
parent 4a8df53553
commit 71caf88ec1
7 changed files with 86 additions and 58 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewRule.java
View File

@ -179,7 +179,12 @@ public abstract class AbstractMaterializedViewRule implements ExplorationRuleFac
return rewriteResults;
}
for (RelationMapping queryToViewTableMapping : queryToViewTableMappings) {
SlotMapping queryToViewSlotMapping = SlotMapping.generate(queryToViewTableMapping);
SlotMapping queryToViewSlotMapping =
materializationContext.getSlotMappingFromCache(queryToViewTableMapping);
if (queryToViewSlotMapping == null) {
queryToViewSlotMapping = SlotMapping.generate(queryToViewTableMapping);
materializationContext.addSlotMappingToCache(queryToViewTableMapping, queryToViewSlotMapping);
}
if (queryToViewSlotMapping == null) {
materializationContext.recordFailReason(queryStructInfo,
"Query to view slot mapping is null", () -> "");
@ -187,7 +192,7 @@ public abstract class AbstractMaterializedViewRule implements ExplorationRuleFac
}
SlotMapping viewToQuerySlotMapping = queryToViewSlotMapping.inverse();
LogicalCompatibilityContext compatibilityContext = LogicalCompatibilityContext.from(
queryToViewTableMapping, queryToViewSlotMapping, queryStructInfo, viewStructInfo);
queryToViewTableMapping, viewToQuerySlotMapping, queryStructInfo, viewStructInfo);
ComparisonResult comparisonResult = StructInfo.isGraphLogicalEquals(queryStructInfo, viewStructInfo,
compatibilityContext);
if (comparisonResult.isInvalid()) {

7
fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/LogicalCompatibilityContext.java
View File

@ -128,7 +128,7 @@ public class LogicalCompatibilityContext {
* this make expression mapping between query and view by relation and the slot in relation mapping
*/
public static LogicalCompatibilityContext from(RelationMapping relationMapping,
SlotMapping queryToViewSlotMapping,
SlotMapping viewToQuerySlotMapping,
StructInfo queryStructInfo,
StructInfo viewStructInfo) {
// init node mapping
@ -147,11 +147,8 @@ public class LogicalCompatibilityContext {
queryToViewNodeMapping.put(queryStructInfoNode, viewStructInfoNode);
}
}
// init expression mapping
Map<SlotReference, SlotReference> viewToQuerySlotMapping = queryToViewSlotMapping.inverse()
.toSlotReferenceMap();
return new LogicalCompatibilityContext(queryToViewNodeMapping,
viewToQuerySlotMapping,
viewToQuerySlotMapping.toSlotReferenceMap(),
queryStructInfo,
viewStructInfo);
}

13
fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializationContext.java
View File

@ -23,6 +23,8 @@ import org.apache.doris.common.Pair;
import org.apache.doris.nereids.CascadesContext;
import org.apache.doris.nereids.memo.GroupId;
import org.apache.doris.nereids.rules.exploration.mv.mapping.ExpressionMapping;
import org.apache.doris.nereids.rules.exploration.mv.mapping.RelationMapping;
import org.apache.doris.nereids.rules.exploration.mv.mapping.SlotMapping;
import org.apache.doris.nereids.trees.expressions.Expression;
import org.apache.doris.nereids.trees.plans.ObjectId;
import org.apache.doris.nereids.trees.plans.Plan;
@ -40,8 +42,10 @@ import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import java.util.BitSet;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
import java.util.function.Supplier;
@ -52,6 +56,7 @@ import java.util.stream.Collectors;
*/
public abstract class MaterializationContext {
private static final Logger LOG = LogManager.getLogger(MaterializationContext.class);
public final Map<RelationMapping, SlotMapping> queryToMvSlotMappingCache = new HashMap<>();
protected List<Table> baseTables;
protected List<Table> baseViews;
// The plan of mv def sql
@ -140,6 +145,14 @@ public abstract class MaterializationContext {
}
}
public void addSlotMappingToCache(RelationMapping relationMapping, SlotMapping slotMapping) {
queryToMvSlotMappingCache.put(relationMapping, slotMapping);
}
public SlotMapping getSlotMappingFromCache(RelationMapping relationMapping) {
return queryToMvSlotMappingCache.get(relationMapping);
}
/**
* Try to generate scan plan for materialization
* if MaterializationContext is already rewritten successfully, then should generate new scan plan in later

23
fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/mapping/ExpressionMapping.java
View File

@ -23,7 +23,6 @@ import org.apache.doris.nereids.util.ExpressionUtils;
import org.apache.doris.nereids.util.Utils;
import com.google.common.collect.ArrayListMultimap;
import com.google.common.collect.ImmutableMultimap;
import com.google.common.collect.Lists;
import com.google.common.collect.Multimap;
@ -32,7 +31,6 @@ import java.util.Collection;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
/**
* Expression mapping, maybe one expression map to multi expression
@ -104,27 +102,6 @@ public class ExpressionMapping extends Mapping {
return new ExpressionMapping(expressionMultiMap);
}
@Override
public Mapping chainedFold(Mapping target) {
ImmutableMultimap.Builder<Expression, Expression> foldedMappingBuilder =
ImmutableMultimap.builder();
Multimap<Expression, Expression> targetMapping
= ((ExpressionMapping) target).getExpressionMapping();
for (Entry<Expression, ? extends Collection<Expression>> exprMapping :
this.getExpressionMapping().asMap().entrySet()) {
Collection<? extends Expression> valueExpressions = exprMapping.getValue();
valueExpressions.forEach(valueExpr -> {
if (targetMapping.containsKey(valueExpr)) {
targetMapping.get(valueExpr).forEach(
targetValue -> foldedMappingBuilder.put(exprMapping.getKey(), targetValue));
}
});
}
return new ExpressionMapping(foldedMappingBuilder.build());
}
@Override
public String toString() {
return Utils.toSqlString("ExpressionMapping", "expressionMapping", expressionMapping);

24
fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/mapping/Mapping.java
View File

@ -22,6 +22,8 @@ import org.apache.doris.nereids.trees.expressions.Slot;
import org.apache.doris.nereids.trees.plans.RelationId;
import org.apache.doris.nereids.trees.plans.algebra.CatalogRelation;
import java.util.HashMap;
import java.util.Map;
import java.util.Objects;
import javax.annotation.Nullable;
@ -38,10 +40,15 @@ public abstract class Mapping {
public final RelationId relationId;
public final CatalogRelation belongedRelation;
// Generate eagerly, will be used to generate slot mapping
private final Map<String, Slot> slotNameToSlotMap = new HashMap<>();
public MappedRelation(RelationId relationId, CatalogRelation belongedRelation) {
this.relationId = relationId;
this.belongedRelation = belongedRelation;
for (Slot slot : belongedRelation.getOutput()) {
slotNameToSlotMap.put(slot.getName(), slot);
}
}
public static MappedRelation of(RelationId relationId, CatalogRelation belongedRelation) {
@ -56,6 +63,10 @@ public abstract class Mapping {
return belongedRelation;
}
public Map<String, Slot> getSlotNameToSlotMap() {
return slotNameToSlotMap;
}
@Override
public boolean equals(Object o) {
if (this == o) {
@ -72,6 +83,11 @@ public abstract class Mapping {
public int hashCode() {
return Objects.hash(relationId);
}
@Override
public String toString() {
return "MappedRelation{" + "relationId=" + relationId + ", slotNameToSlotMap=" + slotNameToSlotMap + '}';
}
}
/**
@ -141,12 +157,4 @@ public abstract class Mapping {
return "MappedSlot{" + "slot=" + slot + '}';
}
}
/** Chain fold tow mapping, such as this mapping is {[a -> b]}, the target mapping is
* {[b -> c]} after chain fold, this result will be {[a -> c]}, if the value side in this mapping
* can get the key in the target mapping, will lose the mapping
*/
protected Mapping chainedFold(Mapping target) {
return null;
}
}

18
fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/mapping/RelationMapping.java
View File

@ -32,6 +32,7 @@ import com.google.common.collect.Sets;
import java.util.ArrayList;
import java.util.List;
import java.util.Objects;
import java.util.Set;
/**
@ -143,4 +144,21 @@ public class RelationMapping extends Mapping {
private static Long getTableQualifier(TableIf tableIf) {
return tableIf.getId();
}
@Override
public boolean equals(Object o) {
if (this == o) {
return true;
}
if (o == null || getClass() != o.getClass()) {
return false;
}
RelationMapping that = (RelationMapping) o;
return Objects.equals(mappedRelationMap, that.mappedRelationMap);
}
@Override
public int hashCode() {
return Objects.hash(mappedRelationMap);
}
}

50
fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/mapping/SlotMapping.java
View File

@ -22,10 +22,11 @@ import org.apache.doris.nereids.trees.expressions.SlotReference;
import com.google.common.collect.BiMap;
import com.google.common.collect.HashBiMap;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import java.util.HashMap;
import java.util.Map;
import java.util.stream.Collectors;
import javax.annotation.Nullable;
/**
@ -33,11 +34,15 @@ import javax.annotation.Nullable;
*/
public class SlotMapping extends Mapping {
public static final Logger LOG = LogManager.getLogger(SlotMapping.class);
private final BiMap<MappedSlot, MappedSlot> relationSlotMap;
private Map<SlotReference, SlotReference> slotReferenceMap;
public SlotMapping(BiMap<MappedSlot, MappedSlot> relationSlotMap) {
public SlotMapping(BiMap<MappedSlot, MappedSlot> relationSlotMap,
Map<SlotReference, SlotReference> slotReferenceMap) {
this.relationSlotMap = relationSlotMap;
this.slotReferenceMap = slotReferenceMap;
}
public BiMap<MappedSlot, MappedSlot> getRelationSlotMap() {
@ -45,11 +50,12 @@ public class SlotMapping extends Mapping {
}
public SlotMapping inverse() {
return SlotMapping.of(relationSlotMap.inverse());
return SlotMapping.of(relationSlotMap.inverse(), null);
}
public static SlotMapping of(BiMap<MappedSlot, MappedSlot> relationSlotMap) {
return new SlotMapping(relationSlotMap);
public static SlotMapping of(BiMap<MappedSlot, MappedSlot> relationSlotMap,
Map<SlotReference, SlotReference> slotReferenceMap) {
return new SlotMapping(relationSlotMap, slotReferenceMap);
}
/**
@ -58,26 +64,31 @@ public class SlotMapping extends Mapping {
@Nullable
public static SlotMapping generate(RelationMapping relationMapping) {
BiMap<MappedSlot, MappedSlot> relationSlotMap = HashBiMap.create();
Map<SlotReference, SlotReference> slotReferenceMap = new HashMap<>();
BiMap<MappedRelation, MappedRelation> mappedRelationMap = relationMapping.getMappedRelationMap();
for (Map.Entry<MappedRelation, MappedRelation> mappedRelationEntry : mappedRelationMap.entrySet()) {
Map<String, Slot> targetNameSlotMap =
mappedRelationEntry.getValue().getBelongedRelation().getOutput().stream()
.collect(Collectors.toMap(Slot::getName, slot -> slot));
for (Slot sourceSlot : mappedRelationEntry.getKey().getBelongedRelation().getOutput()) {
Slot targetSlot = targetNameSlotMap.get(sourceSlot.getName());
MappedRelation sourceRelation = mappedRelationEntry.getKey();
Map<String, Slot> sourceSlotNameToSlotMap = sourceRelation.getSlotNameToSlotMap();
MappedRelation targetRelation = mappedRelationEntry.getValue();
Map<String, Slot> targetSlotNameSlotMap = targetRelation.getSlotNameToSlotMap();
for (String sourceSlotName : sourceSlotNameToSlotMap.keySet()) {
Slot targetSlot = targetSlotNameSlotMap.get(sourceSlotName);
// source slot can not map from target, bail out
if (targetSlot == null) {
LOG.warn(String.format("SlotMapping generate is null, source relation is %s, "
+ "target relation is %s", sourceRelation, targetRelation));
return null;
}
relationSlotMap.put(MappedSlot.of(sourceSlot, mappedRelationEntry.getKey().getBelongedRelation()),
MappedSlot.of(targetSlot, mappedRelationEntry.getValue().getBelongedRelation()));
Slot sourceSlot = sourceSlotNameToSlotMap.get(sourceSlotName);
relationSlotMap.put(MappedSlot.of(sourceSlot,
sourceRelation.getBelongedRelation()),
MappedSlot.of(targetSlot, targetRelation.getBelongedRelation()));
slotReferenceMap.put((SlotReference) sourceSlot, (SlotReference) targetSlot);
}
}
return SlotMapping.of(relationSlotMap);
}
public Map<MappedSlot, MappedSlot> toMappedSlotMap() {
return (Map) this.getRelationSlotMap();
return SlotMapping.of(relationSlotMap, slotReferenceMap);
}
/**
@ -87,12 +98,11 @@ public class SlotMapping extends Mapping {
if (this.slotReferenceMap != null) {
return this.slotReferenceMap;
}
Map<SlotReference, SlotReference> slotReferenceSlotReferenceMap = new HashMap<>();
this.slotReferenceMap = new HashMap<>();
for (Map.Entry<MappedSlot, MappedSlot> entry : this.getRelationSlotMap().entrySet()) {
slotReferenceSlotReferenceMap.put((SlotReference) entry.getKey().getSlot(),
this.slotReferenceMap.put((SlotReference) entry.getKey().getSlot(),
(SlotReference) entry.getValue().getSlot());
}
this.slotReferenceMap = slotReferenceSlotReferenceMap;
return this.slotReferenceMap;
}